src/lttng-abi.c

   1 /* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only)
   2  *
   3  * lttng-abi.c
   4  *
   5  * LTTng ABI
   6  *
   7  * Copyright (C) 2010-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   8  *
   9  * Mimic system calls for:
  10  * - session creation, returns a file descriptor or failure.
  11  *   - channel creation, returns a file descriptor or failure.
  12  *     - Operates on a session file descriptor
  13  *     - Takes all channel options as parameters.
  14  *   - stream get, returns a file descriptor or failure.
  15  *     - Operates on a channel file descriptor.
  16  *   - stream notifier get, returns a file descriptor or failure.
  17  *     - Operates on a channel file descriptor.
  18  *   - event creation, returns a file descriptor or failure.
  19  *     - Operates on a channel file descriptor
  20  *     - Takes an event name as parameter
  21  *     - Takes an instrumentation source as parameter
  22  *       - e.g. tracepoints, dynamic_probes...
  23  *     - Takes instrumentation source specific arguments.
  24  */
  25
  26 #include <linux/module.h>
  27 #include <linux/proc_fs.h>
  28 #include <linux/anon_inodes.h>
  29 #include <linux/file.h>
  30 #include <linux/uaccess.h>
  31 #include <linux/slab.h>
  32 #include <linux/err.h>
  33 #include <wrapper/vmalloc.h>    /* for wrapper_vmalloc_sync_mappings() */
  34 #include <ringbuffer/vfs.h>
  35 #include <ringbuffer/backend.h>
  36 #include <ringbuffer/frontend.h>
  37 #include <wrapper/compiler_attributes.h>
  38 #include <wrapper/poll.h>
  39 #include <wrapper/file.h>
  40 #include <wrapper/kref.h>
  41 #include <wrapper/barrier.h>
  42 #include <lttng/string-utils.h>
  43 #include <lttng/abi.h>
  44 #include <lttng/abi-old.h>
  45 #include <lttng/events.h>
  46 #include <lttng/events-internal.h>
  47 #include <lttng/tracer.h>
  48 #include <lttng/tp-mempool.h>
  49 #include <ringbuffer/frontend_types.h>
  50 #include <ringbuffer/iterator.h>
  51
  52 /*
  53  * This is LTTng's own personal way to create a system call as an external
  54  * module. We use ioctl() on /proc/lttng.
  55  */
  56
  57 static struct proc_dir_entry *lttng_proc_dentry;
  58
  59 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,6,0))
  60 static const struct proc_ops lttng_proc_ops;
  61 #else
  62 static const struct file_operations lttng_proc_ops;
  63 #endif
  64
  65 static const struct file_operations lttng_session_fops;
  66 static const struct file_operations lttng_event_notifier_group_fops;
  67 static const struct file_operations lttng_channel_fops;
  68 static const struct file_operations lttng_metadata_fops;
  69 static const struct file_operations lttng_event_recorder_event_fops;
  70 static const struct file_operations lttng_event_recorder_enabler_fops;
  71 static struct file_operations lttng_stream_ring_buffer_file_operations;
  72
  73 static int put_u64(uint64_t val, unsigned long arg);
  74 static int put_u32(uint32_t val, unsigned long arg);
  75
  76 static int validate_zeroed_padding(char *p, size_t len)
  77 {
  78         size_t i;
  79
  80         for (i = 0; i < len; i++) {
  81                 if (p[i])
  82                         return -1;
  83         }
  84         return 0;
  85 }
  86
  87 /*
  88  * Teardown management: opened file descriptors keep a refcount on the module,
  89  * so it can only exit when all file descriptors are closed.
  90  */
  91
  92 static
  93 int lttng_abi_create_session(void)
  94 {
  95         struct lttng_kernel_session *session;
  96         struct file *session_file;
  97         int session_fd, ret;
  98
  99         session = lttng_session_create();
 100         if (!session)
 101                 return -ENOMEM;
 102         session_fd = lttng_get_unused_fd();
 103         if (session_fd < 0) {
 104                 ret = session_fd;
 105                 goto fd_error;
 106         }
 107         session_file = anon_inode_getfile("[lttng_session]",
 108                                           &lttng_session_fops,
 109                                           session, O_RDWR);
 110         if (IS_ERR(session_file)) {
 111                 ret = PTR_ERR(session_file);
 112                 goto file_error;
 113         }
 114         session->priv->file = session_file;
 115         fd_install(session_fd, session_file);
 116         return session_fd;
 117
 118 file_error:
 119         put_unused_fd(session_fd);
 120 fd_error:
 121         lttng_session_destroy(session);
 122         return ret;
 123 }
 124
 125 void event_notifier_send_notification_work_wakeup(struct irq_work *entry)
 126 {
 127         struct lttng_event_notifier_group *event_notifier_group =
 128                         container_of(entry, struct lttng_event_notifier_group,
 129                                         wakeup_pending);
 130         wake_up_interruptible(&event_notifier_group->read_wait);
 131 }
 132
 133 static
 134 int lttng_abi_create_event_notifier_group(void)
 135 {
 136         struct lttng_event_notifier_group *event_notifier_group;
 137         struct file *event_notifier_group_file;
 138         int event_notifier_group_fd, ret;
 139
 140         event_notifier_group = lttng_event_notifier_group_create();
 141         if (!event_notifier_group)
 142                 return -ENOMEM;
 143
 144         event_notifier_group_fd = lttng_get_unused_fd();
 145         if (event_notifier_group_fd < 0) {
 146                 ret = event_notifier_group_fd;
 147                 goto fd_error;
 148         }
 149         event_notifier_group_file = anon_inode_getfile("[lttng_event_notifier_group]",
 150                                           &lttng_event_notifier_group_fops,
 151                                           event_notifier_group, O_RDWR);
 152         if (IS_ERR(event_notifier_group_file)) {
 153                 ret = PTR_ERR(event_notifier_group_file);
 154                 goto file_error;
 155         }
 156
 157         event_notifier_group->file = event_notifier_group_file;
 158         init_waitqueue_head(&event_notifier_group->read_wait);
 159         init_irq_work(&event_notifier_group->wakeup_pending,
 160                       event_notifier_send_notification_work_wakeup);
 161         fd_install(event_notifier_group_fd, event_notifier_group_file);
 162         return event_notifier_group_fd;
 163
 164 file_error:
 165         put_unused_fd(event_notifier_group_fd);
 166 fd_error:
 167         lttng_event_notifier_group_destroy(event_notifier_group);
 168         return ret;
 169 }
 170
 171 static
 172 int lttng_abi_tracepoint_list(void)
 173 {
 174         struct file *tracepoint_list_file;
 175         int file_fd, ret;
 176
 177         file_fd = lttng_get_unused_fd();
 178         if (file_fd < 0) {
 179                 ret = file_fd;
 180                 goto fd_error;
 181         }
 182
 183         tracepoint_list_file = anon_inode_getfile("[lttng_tracepoint_list]",
 184                                           &lttng_tracepoint_list_fops,
 185                                           NULL, O_RDWR);
 186         if (IS_ERR(tracepoint_list_file)) {
 187                 ret = PTR_ERR(tracepoint_list_file);
 188                 goto file_error;
 189         }
 190         ret = lttng_tracepoint_list_fops.open(NULL, tracepoint_list_file);
 191         if (ret < 0)
 192                 goto open_error;
 193         fd_install(file_fd, tracepoint_list_file);
 194         return file_fd;
 195
 196 open_error:
 197         fput(tracepoint_list_file);
 198 file_error:
 199         put_unused_fd(file_fd);
 200 fd_error:
 201         return ret;
 202 }
 203
 204 #ifndef CONFIG_HAVE_SYSCALL_TRACEPOINTS
 205 static inline
 206 int lttng_abi_syscall_list(void)
 207 {
 208         return -ENOSYS;
 209 }
 210 #else
 211 static
 212 int lttng_abi_syscall_list(void)
 213 {
 214         struct file *syscall_list_file;
 215         int file_fd, ret;
 216
 217         file_fd = lttng_get_unused_fd();
 218         if (file_fd < 0) {
 219                 ret = file_fd;
 220                 goto fd_error;
 221         }
 222
 223         syscall_list_file = anon_inode_getfile("[lttng_syscall_list]",
 224                                           &lttng_syscall_list_fops,
 225                                           NULL, O_RDWR);
 226         if (IS_ERR(syscall_list_file)) {
 227                 ret = PTR_ERR(syscall_list_file);
 228                 goto file_error;
 229         }
 230         ret = lttng_syscall_list_fops.open(NULL, syscall_list_file);
 231         if (ret < 0)
 232                 goto open_error;
 233         fd_install(file_fd, syscall_list_file);
 234         return file_fd;
 235
 236 open_error:
 237         fput(syscall_list_file);
 238 file_error:
 239         put_unused_fd(file_fd);
 240 fd_error:
 241         return ret;
 242 }
 243 #endif
 244
 245 static
 246 void lttng_abi_tracer_version(struct lttng_kernel_abi_tracer_version *v)
 247 {
 248         v->major = LTTNG_MODULES_MAJOR_VERSION;
 249         v->minor = LTTNG_MODULES_MINOR_VERSION;
 250         v->patchlevel = LTTNG_MODULES_PATCHLEVEL_VERSION;
 251 }
 252
 253 static
 254 void lttng_abi_tracer_abi_version(struct lttng_kernel_abi_tracer_abi_version *v)
 255 {
 256         v->major = LTTNG_KERNEL_ABI_MAJOR_VERSION;
 257         v->minor = LTTNG_KERNEL_ABI_MINOR_VERSION;
 258 }
 259
 260 static
 261 long lttng_abi_add_context(struct file *file,
 262         struct lttng_kernel_abi_context *context_param,
 263         struct lttng_kernel_ctx **ctx, struct lttng_kernel_session *session)
 264 {
 265
 266         if (session->priv->been_active)
 267                 return -EPERM;
 268
 269         switch (context_param->ctx) {
 270         case LTTNG_KERNEL_ABI_CONTEXT_PID:
 271                 return lttng_add_pid_to_ctx(ctx);
 272         case LTTNG_KERNEL_ABI_CONTEXT_PRIO:
 273                 return lttng_add_prio_to_ctx(ctx);
 274         case LTTNG_KERNEL_ABI_CONTEXT_NICE:
 275                 return lttng_add_nice_to_ctx(ctx);
 276         case LTTNG_KERNEL_ABI_CONTEXT_VPID:
 277                 return lttng_add_vpid_to_ctx(ctx);
 278         case LTTNG_KERNEL_ABI_CONTEXT_TID:
 279                 return lttng_add_tid_to_ctx(ctx);
 280         case LTTNG_KERNEL_ABI_CONTEXT_VTID:
 281                 return lttng_add_vtid_to_ctx(ctx);
 282         case LTTNG_KERNEL_ABI_CONTEXT_PPID:
 283                 return lttng_add_ppid_to_ctx(ctx);
 284         case LTTNG_KERNEL_ABI_CONTEXT_VPPID:
 285                 return lttng_add_vppid_to_ctx(ctx);
 286         case LTTNG_KERNEL_ABI_CONTEXT_PERF_COUNTER:
 287                 context_param->u.perf_counter.name[LTTNG_KERNEL_ABI_SYM_NAME_LEN - 1] = '\0';
 288                 return lttng_add_perf_counter_to_ctx(context_param->u.perf_counter.type,
 289                                 context_param->u.perf_counter.config,
 290                                 context_param->u.perf_counter.name,
 291                                 ctx);
 292         case LTTNG_KERNEL_ABI_CONTEXT_PROCNAME:
 293                 return lttng_add_procname_to_ctx(ctx);
 294         case LTTNG_KERNEL_ABI_CONTEXT_HOSTNAME:
 295                 return lttng_add_hostname_to_ctx(ctx);
 296         case LTTNG_KERNEL_ABI_CONTEXT_CPU_ID:
 297                 return lttng_add_cpu_id_to_ctx(ctx);
 298         case LTTNG_KERNEL_ABI_CONTEXT_INTERRUPTIBLE:
 299                 return lttng_add_interruptible_to_ctx(ctx);
 300         case LTTNG_KERNEL_ABI_CONTEXT_NEED_RESCHEDULE:
 301                 return lttng_add_need_reschedule_to_ctx(ctx);
 302         case LTTNG_KERNEL_ABI_CONTEXT_PREEMPTIBLE:
 303                 return lttng_add_preemptible_to_ctx(ctx);
 304         case LTTNG_KERNEL_ABI_CONTEXT_MIGRATABLE:
 305                 return lttng_add_migratable_to_ctx(ctx);
 306         case LTTNG_KERNEL_ABI_CONTEXT_CALLSTACK_KERNEL:
 307         case LTTNG_KERNEL_ABI_CONTEXT_CALLSTACK_USER:
 308                 return lttng_add_callstack_to_ctx(ctx, context_param->ctx);
 309         case LTTNG_KERNEL_ABI_CONTEXT_CGROUP_NS:
 310                 return lttng_add_cgroup_ns_to_ctx(ctx);
 311         case LTTNG_KERNEL_ABI_CONTEXT_IPC_NS:
 312                 return lttng_add_ipc_ns_to_ctx(ctx);
 313         case LTTNG_KERNEL_ABI_CONTEXT_MNT_NS:
 314                 return lttng_add_mnt_ns_to_ctx(ctx);
 315         case LTTNG_KERNEL_ABI_CONTEXT_NET_NS:
 316                 return lttng_add_net_ns_to_ctx(ctx);
 317         case LTTNG_KERNEL_ABI_CONTEXT_PID_NS:
 318                 return lttng_add_pid_ns_to_ctx(ctx);
 319         case LTTNG_KERNEL_ABI_CONTEXT_USER_NS:
 320                 return lttng_add_user_ns_to_ctx(ctx);
 321         case LTTNG_KERNEL_ABI_CONTEXT_UTS_NS:
 322                 return lttng_add_uts_ns_to_ctx(ctx);
 323         case LTTNG_KERNEL_ABI_CONTEXT_UID:
 324                 return lttng_add_uid_to_ctx(ctx);
 325         case LTTNG_KERNEL_ABI_CONTEXT_EUID:
 326                 return lttng_add_euid_to_ctx(ctx);
 327         case LTTNG_KERNEL_ABI_CONTEXT_SUID:
 328                 return lttng_add_suid_to_ctx(ctx);
 329         case LTTNG_KERNEL_ABI_CONTEXT_GID:
 330                 return lttng_add_gid_to_ctx(ctx);
 331         case LTTNG_KERNEL_ABI_CONTEXT_EGID:
 332                 return lttng_add_egid_to_ctx(ctx);
 333         case LTTNG_KERNEL_ABI_CONTEXT_SGID:
 334                 return lttng_add_sgid_to_ctx(ctx);
 335         case LTTNG_KERNEL_ABI_CONTEXT_VUID:
 336                 return lttng_add_vuid_to_ctx(ctx);
 337         case LTTNG_KERNEL_ABI_CONTEXT_VEUID:
 338                 return lttng_add_veuid_to_ctx(ctx);
 339         case LTTNG_KERNEL_ABI_CONTEXT_VSUID:
 340                 return lttng_add_vsuid_to_ctx(ctx);
 341         case LTTNG_KERNEL_ABI_CONTEXT_VGID:
 342                 return lttng_add_vgid_to_ctx(ctx);
 343         case LTTNG_KERNEL_ABI_CONTEXT_VEGID:
 344                 return lttng_add_vegid_to_ctx(ctx);
 345         case LTTNG_KERNEL_ABI_CONTEXT_VSGID:
 346                 return lttng_add_vsgid_to_ctx(ctx);
 347         case LTTNG_KERNEL_ABI_CONTEXT_TIME_NS:
 348                 return lttng_add_time_ns_to_ctx(ctx);
 349         default:
 350                 return -EINVAL;
 351         }
 352 }
 353
 354 /**
 355  *      lttng_ioctl - lttng syscall through ioctl
 356  *
 357  *      @file: the file
 358  *      @cmd: the command
 359  *      @arg: command arg
 360  *
 361  *      This ioctl implements lttng commands:
 362  *      LTTNG_KERNEL_ABI_SESSION
 363  *              Returns a LTTng trace session file descriptor
 364  *      LTTNG_KERNEL_ABI_TRACER_VERSION
 365  *              Returns the LTTng kernel tracer version
 366  *      LTTNG_KERNEL_ABI_TRACEPOINT_LIST
 367  *              Returns a file descriptor listing available tracepoints
 368  *      LTTNG_KERNEL_ABI_WAIT_QUIESCENT
 369  *              Returns after all previously running probes have completed
 370  *      LTTNG_KERNEL_ABI_TRACER_ABI_VERSION
 371  *              Returns the LTTng kernel tracer ABI version
 372  *      LTTNG_KERNEL_ABI_EVENT_NOTIFIER_GROUP_CREATE
 373  *              Returns a LTTng event notifier group file descriptor
 374  *
 375  * The returned session will be deleted when its file descriptor is closed.
 376  */
 377 static
 378 long lttng_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 379 {
 380         switch (cmd) {
 381         case LTTNG_KERNEL_ABI_OLD_SESSION:
 382         case LTTNG_KERNEL_ABI_SESSION:
 383                 return lttng_abi_create_session();
 384         case LTTNG_KERNEL_ABI_EVENT_NOTIFIER_GROUP_CREATE:
 385                 return lttng_abi_create_event_notifier_group();
 386         case LTTNG_KERNEL_ABI_OLD_TRACER_VERSION:
 387         {
 388                 struct lttng_kernel_abi_tracer_version v;
 389                 struct lttng_kernel_abi_old_tracer_version oldv;
 390                 struct lttng_kernel_abi_old_tracer_version *uversion =
 391                         (struct lttng_kernel_abi_old_tracer_version __user *) arg;
 392
 393                 lttng_abi_tracer_version(&v);
 394                 oldv.major = v.major;
 395                 oldv.minor = v.minor;
 396                 oldv.patchlevel = v.patchlevel;
 397
 398                 if (copy_to_user(uversion, &oldv, sizeof(oldv)))
 399                         return -EFAULT;
 400                 return 0;
 401         }
 402         case LTTNG_KERNEL_ABI_TRACER_VERSION:
 403         {
 404                 struct lttng_kernel_abi_tracer_version version;
 405                 struct lttng_kernel_abi_tracer_version *uversion =
 406                         (struct lttng_kernel_abi_tracer_version __user *) arg;
 407
 408                 lttng_abi_tracer_version(&version);
 409
 410                 if (copy_to_user(uversion, &version, sizeof(version)))
 411                         return -EFAULT;
 412                 return 0;
 413         }
 414         case LTTNG_KERNEL_ABI_TRACER_ABI_VERSION:
 415         {
 416                 struct lttng_kernel_abi_tracer_abi_version version;
 417                 struct lttng_kernel_abi_tracer_abi_version *uversion =
 418                         (struct lttng_kernel_abi_tracer_abi_version __user *) arg;
 419
 420                 lttng_abi_tracer_abi_version(&version);
 421
 422                 if (copy_to_user(uversion, &version, sizeof(version)))
 423                         return -EFAULT;
 424                 return 0;
 425         }
 426         case LTTNG_KERNEL_ABI_OLD_TRACEPOINT_LIST:
 427         case LTTNG_KERNEL_ABI_TRACEPOINT_LIST:
 428                 return lttng_abi_tracepoint_list();
 429         case LTTNG_KERNEL_ABI_SYSCALL_LIST:
 430                 return lttng_abi_syscall_list();
 431         case LTTNG_KERNEL_ABI_OLD_WAIT_QUIESCENT:
 432         case LTTNG_KERNEL_ABI_WAIT_QUIESCENT:
 433                 synchronize_trace();
 434                 return 0;
 435         case LTTNG_KERNEL_ABI_OLD_CALIBRATE:
 436         {
 437                 struct lttng_kernel_abi_old_calibrate __user *ucalibrate =
 438                         (struct lttng_kernel_abi_old_calibrate __user *) arg;
 439                 struct lttng_kernel_abi_old_calibrate old_calibrate;
 440                 struct lttng_kernel_abi_calibrate calibrate;
 441                 int ret;
 442
 443                 if (copy_from_user(&old_calibrate, ucalibrate, sizeof(old_calibrate)))
 444                         return -EFAULT;
 445                 calibrate.type = old_calibrate.type;
 446                 ret = lttng_calibrate(&calibrate);
 447                 if (copy_to_user(ucalibrate, &old_calibrate, sizeof(old_calibrate)))
 448                         return -EFAULT;
 449                 return ret;
 450         }
 451         case LTTNG_KERNEL_ABI_CALIBRATE:
 452         {
 453                 struct lttng_kernel_abi_calibrate __user *ucalibrate =
 454                         (struct lttng_kernel_abi_calibrate __user *) arg;
 455                 struct lttng_kernel_abi_calibrate calibrate;
 456                 int ret;
 457
 458                 if (copy_from_user(&calibrate, ucalibrate, sizeof(calibrate)))
 459                         return -EFAULT;
 460                 ret = lttng_calibrate(&calibrate);
 461                 if (copy_to_user(ucalibrate, &calibrate, sizeof(calibrate)))
 462                         return -EFAULT;
 463                 return ret;
 464         }
 465         default:
 466                 return -ENOIOCTLCMD;
 467         }
 468 }
 469
 470 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(5,6,0))
 471 static const struct proc_ops lttng_proc_ops = {
 472         .proc_ioctl = lttng_ioctl,
 473 #ifdef CONFIG_COMPAT
 474         .proc_compat_ioctl = lttng_ioctl,
 475 #endif /* CONFIG_COMPAT */
 476 };
 477 #else
 478 static const struct file_operations lttng_proc_ops = {
 479         .owner = THIS_MODULE,
 480         .unlocked_ioctl = lttng_ioctl,
 481 #ifdef CONFIG_COMPAT
 482         .compat_ioctl = lttng_ioctl,
 483 #endif /* CONFIG_COMPAT */
 484 };
 485 #endif
 486
 487 static
 488 int lttng_abi_create_channel(struct file *session_file,
 489                              struct lttng_kernel_abi_channel *chan_param,
 490                              enum channel_type channel_type)
 491 {
 492         struct lttng_kernel_session *session = session_file->private_data;
 493         const struct file_operations *fops = NULL;
 494         const char *transport_name;
 495         struct lttng_kernel_channel_buffer *chan;
 496         struct file *chan_file;
 497         int chan_fd;
 498         int ret = 0;
 499
 500         chan_fd = lttng_get_unused_fd();
 501         if (chan_fd < 0) {
 502                 ret = chan_fd;
 503                 goto fd_error;
 504         }
 505         switch (channel_type) {
 506         case PER_CPU_CHANNEL:
 507                 fops = &lttng_channel_fops;
 508                 break;
 509         case METADATA_CHANNEL:
 510                 fops = &lttng_metadata_fops;
 511                 break;
 512         }
 513
 514         chan_file = anon_inode_getfile("[lttng_channel]",
 515                                        fops,
 516                                        NULL, O_RDWR);
 517         if (IS_ERR(chan_file)) {
 518                 ret = PTR_ERR(chan_file);
 519                 goto file_error;
 520         }
 521         switch (channel_type) {
 522         case PER_CPU_CHANNEL:
 523                 if (chan_param->output == LTTNG_KERNEL_ABI_SPLICE) {
 524                         transport_name = chan_param->overwrite ?
 525                                 "relay-overwrite" : "relay-discard";
 526                 } else if (chan_param->output == LTTNG_KERNEL_ABI_MMAP) {
 527                         transport_name = chan_param->overwrite ?
 528                                 "relay-overwrite-mmap" : "relay-discard-mmap";
 529                 } else {
 530                         return -EINVAL;
 531                 }
 532                 break;
 533         case METADATA_CHANNEL:
 534                 if (chan_param->output == LTTNG_KERNEL_ABI_SPLICE)
 535                         transport_name = "relay-metadata";
 536                 else if (chan_param->output == LTTNG_KERNEL_ABI_MMAP)
 537                         transport_name = "relay-metadata-mmap";
 538                 else
 539                         return -EINVAL;
 540                 break;
 541         default:
 542                 transport_name = "<unknown>";
 543                 break;
 544         }
 545         if (!atomic_long_add_unless(&session_file->f_count, 1, LONG_MAX)) {
 546                 ret = -EOVERFLOW;
 547                 goto refcount_error;
 548         }
 549         /*
 550          * We tolerate no failure path after channel creation. It will stay
 551          * invariant for the rest of the session.
 552          */
 553         chan = lttng_channel_create(session, transport_name, NULL,
 554                                   chan_param->subbuf_size,
 555                                   chan_param->num_subbuf,
 556                                   chan_param->switch_timer_interval,
 557                                   chan_param->read_timer_interval,
 558                                   channel_type);
 559         if (!chan) {
 560                 ret = -EINVAL;
 561                 goto chan_error;
 562         }
 563         chan->priv->parent.file = chan_file;
 564         chan_file->private_data = chan;
 565         fd_install(chan_fd, chan_file);
 566
 567         return chan_fd;
 568
 569 chan_error:
 570         atomic_long_dec(&session_file->f_count);
 571 refcount_error:
 572         fput(chan_file);
 573 file_error:
 574         put_unused_fd(chan_fd);
 575 fd_error:
 576         return ret;
 577 }
 578
 579 static
 580 int lttng_abi_session_set_name(struct lttng_kernel_session *session,
 581                 struct lttng_kernel_abi_session_name *name)
 582 {
 583         size_t len;
 584
 585         len = strnlen(name->name, LTTNG_KERNEL_ABI_SESSION_NAME_LEN);
 586
 587         if (len == LTTNG_KERNEL_ABI_SESSION_NAME_LEN) {
 588                 /* Name is too long/malformed */
 589                 return -EINVAL;
 590         }
 591
 592         strcpy(session->priv->name, name->name);
 593         return 0;
 594 }
 595
 596 static
 597 int lttng_abi_session_set_creation_time(struct lttng_kernel_session *session,
 598                 struct lttng_kernel_abi_session_creation_time *time)
 599 {
 600         size_t len;
 601
 602         len = strnlen(time->iso8601, LTTNG_KERNEL_ABI_SESSION_CREATION_TIME_ISO8601_LEN);
 603
 604         if (len == LTTNG_KERNEL_ABI_SESSION_CREATION_TIME_ISO8601_LEN) {
 605                 /* Time is too long/malformed */
 606                 return -EINVAL;
 607         }
 608
 609         strcpy(session->priv->creation_time, time->iso8601);
 610         return 0;
 611 }
 612
 613 static
 614 int lttng_counter_release(struct inode *inode, struct file *file)
 615 {
 616         struct lttng_counter *counter = file->private_data;
 617
 618         if (counter) {
 619                 /*
 620                  * Do not destroy the counter itself. Wait of the owner
 621                  * (event_notifier group) to be destroyed.
 622                  */
 623                 fput(counter->owner);
 624         }
 625
 626         return 0;
 627 }
 628
 629 static
 630 long lttng_counter_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 631 {
 632         struct lttng_counter *counter = file->private_data;
 633         size_t indexes[LTTNG_KERNEL_ABI_COUNTER_DIMENSION_MAX] = { 0 };
 634         int i;
 635
 636         switch (cmd) {
 637         case LTTNG_KERNEL_ABI_COUNTER_READ:
 638         {
 639                 struct lttng_kernel_abi_counter_read local_counter_read;
 640                 struct lttng_kernel_abi_counter_read __user *ucounter_read =
 641                                 (struct lttng_kernel_abi_counter_read __user *) arg;
 642                 bool overflow, underflow;
 643                 int64_t value;
 644                 int32_t cpu;
 645                 int ret;
 646
 647                 if (copy_from_user(&local_counter_read, ucounter_read,
 648                                         sizeof(local_counter_read)))
 649                         return -EFAULT;
 650                 if (validate_zeroed_padding(local_counter_read.padding,
 651                                 sizeof(local_counter_read.padding)))
 652                         return -EINVAL;
 653
 654                 /* Cast all indexes into size_t. */
 655                 for (i = 0; i < local_counter_read.index.number_dimensions; i++)
 656                         indexes[i] = (size_t) local_counter_read.index.dimension_indexes[i];
 657                 cpu = local_counter_read.cpu;
 658
 659                 ret = lttng_kernel_counter_read(counter, indexes, cpu, &value,
 660                                 &overflow, &underflow);
 661                 if (ret)
 662                         return ret;
 663                 local_counter_read.value.value = value;
 664                 local_counter_read.value.overflow = overflow;
 665                 local_counter_read.value.underflow = underflow;
 666
 667                 if (copy_to_user(&ucounter_read->value, &local_counter_read.value,
 668                                         sizeof(local_counter_read.value)))
 669                         return -EFAULT;
 670
 671                 return 0;
 672         }
 673         case LTTNG_KERNEL_ABI_COUNTER_AGGREGATE:
 674         {
 675                 struct lttng_kernel_abi_counter_aggregate local_counter_aggregate;
 676                 struct lttng_kernel_abi_counter_aggregate __user *ucounter_aggregate =
 677                                 (struct lttng_kernel_abi_counter_aggregate __user *) arg;
 678                 bool overflow, underflow;
 679                 int64_t value;
 680                 int ret;
 681
 682                 if (copy_from_user(&local_counter_aggregate, ucounter_aggregate,
 683                                         sizeof(local_counter_aggregate)))
 684                         return -EFAULT;
 685                 if (validate_zeroed_padding(local_counter_aggregate.padding,
 686                                 sizeof(local_counter_aggregate.padding)))
 687                         return -EINVAL;
 688
 689                 /* Cast all indexes into size_t. */
 690                 for (i = 0; i < local_counter_aggregate.index.number_dimensions; i++)
 691                         indexes[i] = (size_t) local_counter_aggregate.index.dimension_indexes[i];
 692
 693                 ret = lttng_kernel_counter_aggregate(counter, indexes, &value,
 694                                 &overflow, &underflow);
 695                 if (ret)
 696                         return ret;
 697                 local_counter_aggregate.value.value = value;
 698                 local_counter_aggregate.value.overflow = overflow;
 699                 local_counter_aggregate.value.underflow = underflow;
 700
 701                 if (copy_to_user(&ucounter_aggregate->value, &local_counter_aggregate.value,
 702                                         sizeof(local_counter_aggregate.value)))
 703                         return -EFAULT;
 704
 705                 return 0;
 706         }
 707         case LTTNG_KERNEL_ABI_COUNTER_CLEAR:
 708         {
 709                 struct lttng_kernel_abi_counter_clear local_counter_clear;
 710                 struct lttng_kernel_abi_counter_clear __user *ucounter_clear =
 711                                 (struct lttng_kernel_abi_counter_clear __user *) arg;
 712
 713                 if (copy_from_user(&local_counter_clear, ucounter_clear,
 714                                         sizeof(local_counter_clear)))
 715                         return -EFAULT;
 716                 if (validate_zeroed_padding(local_counter_clear.padding,
 717                                 sizeof(local_counter_clear.padding)))
 718                         return -EINVAL;
 719
 720                 /* Cast all indexes into size_t. */
 721                 for (i = 0; i < local_counter_clear.index.number_dimensions; i++)
 722                         indexes[i] = (size_t) local_counter_clear.index.dimension_indexes[i];
 723
 724                 return lttng_kernel_counter_clear(counter, indexes);
 725         }
 726         default:
 727                 WARN_ON_ONCE(1);
 728                 return -ENOSYS;
 729         }
 730 }
 731
 732 static const struct file_operations lttng_counter_fops = {
 733         .owner = THIS_MODULE,
 734         .release = lttng_counter_release,
 735         .unlocked_ioctl = lttng_counter_ioctl,
 736 #ifdef CONFIG_COMPAT
 737         .compat_ioctl = lttng_counter_ioctl,
 738 #endif
 739 };
 740
 741
 742 static
 743 enum tracker_type get_tracker_type(struct lttng_kernel_abi_tracker_args *tracker)
 744 {
 745         switch (tracker->type) {
 746         case LTTNG_KERNEL_ABI_TRACKER_PID:
 747                 return TRACKER_PID;
 748         case LTTNG_KERNEL_ABI_TRACKER_VPID:
 749                 return TRACKER_VPID;
 750         case LTTNG_KERNEL_ABI_TRACKER_UID:
 751                 return TRACKER_UID;
 752         case LTTNG_KERNEL_ABI_TRACKER_VUID:
 753                 return TRACKER_VUID;
 754         case LTTNG_KERNEL_ABI_TRACKER_GID:
 755                 return TRACKER_GID;
 756         case LTTNG_KERNEL_ABI_TRACKER_VGID:
 757                 return TRACKER_VGID;
 758         default:
 759                 return TRACKER_UNKNOWN;
 760         }
 761 }
 762
 763 /**
 764  *      lttng_session_ioctl - lttng session fd ioctl
 765  *
 766  *      @file: the file
 767  *      @cmd: the command
 768  *      @arg: command arg
 769  *
 770  *      This ioctl implements lttng commands:
 771  *      LTTNG_KERNEL_ABI_CHANNEL
 772  *              Returns a LTTng channel file descriptor
 773  *      LTTNG_KERNEL_ABI_ENABLE
 774  *              Enables tracing for a session (weak enable)
 775  *      LTTNG_KERNEL_ABI_DISABLE
 776  *              Disables tracing for a session (strong disable)
 777  *      LTTNG_KERNEL_ABI_METADATA
 778  *              Returns a LTTng metadata file descriptor
 779  *      LTTNG_KERNEL_ABI_SESSION_TRACK_PID
 780  *              Add PID to session PID tracker
 781  *      LTTNG_KERNEL_ABI_SESSION_UNTRACK_PID
 782  *              Remove PID from session PID tracker
 783  *      LTTNG_KERNEL_ABI_SESSION_TRACK_ID
 784  *              Add ID to tracker
 785  *      LTTNG_KERNEL_ABI_SESSION_UNTRACK_ID
 786  *              Remove ID from tracker
 787  *
 788  * The returned channel will be deleted when its file descriptor is closed.
 789  */
 790 static
 791 long lttng_session_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 792 {
 793         struct lttng_kernel_session *session = file->private_data;
 794         struct lttng_kernel_abi_channel chan_param;
 795         struct lttng_kernel_abi_old_channel old_chan_param;
 796
 797         /*
 798          * Handle backward compatibility. OLD commands have wrong
 799          * directions, replace them by the correct direction.
 800          */
 801         switch (cmd) {
 802         case LTTNG_KERNEL_ABI_OLD_SESSION_TRACK_PID:
 803                 cmd = LTTNG_KERNEL_ABI_SESSION_TRACK_PID;
 804                 break;
 805         case LTTNG_KERNEL_ABI_OLD_SESSION_UNTRACK_PID:
 806                 cmd = LTTNG_KERNEL_ABI_SESSION_UNTRACK_PID;
 807                 break;
 808         case LTTNG_KERNEL_ABI_OLD_SESSION_TRACK_ID:
 809                 cmd = LTTNG_KERNEL_ABI_SESSION_TRACK_ID;
 810                 break;
 811         case LTTNG_KERNEL_ABI_OLD_SESSION_UNTRACK_ID:
 812                 cmd = LTTNG_KERNEL_ABI_SESSION_UNTRACK_ID;
 813                 break;
 814         case LTTNG_KERNEL_ABI_OLD_SESSION_LIST_TRACKER_IDS:
 815                 cmd = LTTNG_KERNEL_ABI_SESSION_LIST_TRACKER_IDS;
 816                 break;
 817         case LTTNG_KERNEL_ABI_OLD_SESSION_SET_NAME:
 818                 cmd = LTTNG_KERNEL_ABI_SESSION_SET_NAME;
 819                 break;
 820         case LTTNG_KERNEL_ABI_OLD_SESSION_SET_CREATION_TIME:
 821                 cmd = LTTNG_KERNEL_ABI_SESSION_SET_CREATION_TIME;
 822                 break;
 823         default:
 824                 /* Nothing to do. */
 825                 break;
 826         }
 827
 828         switch (cmd) {
 829         case LTTNG_KERNEL_ABI_OLD_CHANNEL:
 830         {
 831                 if (copy_from_user(&old_chan_param,
 832                                 (struct lttng_kernel_abi_old_channel __user *) arg,
 833                                 sizeof(struct lttng_kernel_abi_old_channel)))
 834                         return -EFAULT;
 835                 chan_param.overwrite = old_chan_param.overwrite;
 836                 chan_param.subbuf_size = old_chan_param.subbuf_size;
 837                 chan_param.num_subbuf = old_chan_param.num_subbuf;
 838                 chan_param.switch_timer_interval = old_chan_param.switch_timer_interval;
 839                 chan_param.read_timer_interval = old_chan_param.read_timer_interval;
 840                 chan_param.output = old_chan_param.output;
 841
 842                 return lttng_abi_create_channel(file, &chan_param,
 843                                 PER_CPU_CHANNEL);
 844         }
 845         case LTTNG_KERNEL_ABI_CHANNEL:
 846         {
 847                 if (copy_from_user(&chan_param,
 848                                 (struct lttng_kernel_abi_channel __user *) arg,
 849                                 sizeof(struct lttng_kernel_abi_channel)))
 850                         return -EFAULT;
 851                 return lttng_abi_create_channel(file, &chan_param,
 852                                 PER_CPU_CHANNEL);
 853         }
 854         case LTTNG_KERNEL_ABI_OLD_SESSION_START:
 855         case LTTNG_KERNEL_ABI_OLD_ENABLE:
 856         case LTTNG_KERNEL_ABI_SESSION_START:
 857         case LTTNG_KERNEL_ABI_ENABLE:
 858                 return lttng_session_enable(session);
 859         case LTTNG_KERNEL_ABI_OLD_SESSION_STOP:
 860         case LTTNG_KERNEL_ABI_OLD_DISABLE:
 861         case LTTNG_KERNEL_ABI_SESSION_STOP:
 862         case LTTNG_KERNEL_ABI_DISABLE:
 863                 return lttng_session_disable(session);
 864         case LTTNG_KERNEL_ABI_OLD_METADATA:
 865         {
 866                 if (copy_from_user(&old_chan_param,
 867                                 (struct lttng_kernel_abi_old_channel __user *) arg,
 868                                 sizeof(struct lttng_kernel_abi_old_channel)))
 869                         return -EFAULT;
 870                 chan_param.overwrite = old_chan_param.overwrite;
 871                 chan_param.subbuf_size = old_chan_param.subbuf_size;
 872                 chan_param.num_subbuf = old_chan_param.num_subbuf;
 873                 chan_param.switch_timer_interval = old_chan_param.switch_timer_interval;
 874                 chan_param.read_timer_interval = old_chan_param.read_timer_interval;
 875                 chan_param.output = old_chan_param.output;
 876
 877                 return lttng_abi_create_channel(file, &chan_param,
 878                                 METADATA_CHANNEL);
 879         }
 880         case LTTNG_KERNEL_ABI_METADATA:
 881         {
 882                 if (copy_from_user(&chan_param,
 883                                         (struct lttng_kernel_abi_channel __user *) arg,
 884                                         sizeof(struct lttng_kernel_abi_channel)))
 885                         return -EFAULT;
 886                 return lttng_abi_create_channel(file, &chan_param,
 887                                 METADATA_CHANNEL);
 888         }
 889         case LTTNG_KERNEL_ABI_SESSION_TRACK_PID:
 890                 return lttng_session_track_id(session, TRACKER_PID, (int) arg);
 891         case LTTNG_KERNEL_ABI_SESSION_UNTRACK_PID:
 892                 return lttng_session_untrack_id(session, TRACKER_PID, (int) arg);
 893         case LTTNG_KERNEL_ABI_SESSION_TRACK_ID:
 894         {
 895                 struct lttng_kernel_abi_tracker_args tracker;
 896                 enum tracker_type tracker_type;
 897
 898                 if (copy_from_user(&tracker,
 899                                 (struct lttng_kernel_abi_tracker_args __user *) arg,
 900                                 sizeof(struct lttng_kernel_abi_tracker_args)))
 901                         return -EFAULT;
 902                 tracker_type = get_tracker_type(&tracker);
 903                 if (tracker_type == TRACKER_UNKNOWN)
 904                         return -EINVAL;
 905                 return lttng_session_track_id(session, tracker_type, tracker.id);
 906         }
 907         case LTTNG_KERNEL_ABI_SESSION_UNTRACK_ID:
 908         {
 909                 struct lttng_kernel_abi_tracker_args tracker;
 910                 enum tracker_type tracker_type;
 911
 912                 if (copy_from_user(&tracker,
 913                                 (struct lttng_kernel_abi_tracker_args __user *) arg,
 914                                 sizeof(struct lttng_kernel_abi_tracker_args)))
 915                         return -EFAULT;
 916                 tracker_type = get_tracker_type(&tracker);
 917                 if (tracker_type == TRACKER_UNKNOWN)
 918                         return -EINVAL;
 919                 return lttng_session_untrack_id(session, tracker_type,
 920                                 tracker.id);
 921         }
 922         case LTTNG_KERNEL_ABI_SESSION_LIST_TRACKER_PIDS:
 923                 return lttng_session_list_tracker_ids(session, TRACKER_PID);
 924         case LTTNG_KERNEL_ABI_SESSION_LIST_TRACKER_IDS:
 925         {
 926                 struct lttng_kernel_abi_tracker_args tracker;
 927                 enum tracker_type tracker_type;
 928
 929                 if (copy_from_user(&tracker,
 930                                 (struct lttng_kernel_abi_tracker_args __user *) arg,
 931                                 sizeof(struct lttng_kernel_abi_tracker_args)))
 932                         return -EFAULT;
 933                 tracker_type = get_tracker_type(&tracker);
 934                 if (tracker_type == TRACKER_UNKNOWN)
 935                         return -EINVAL;
 936                 return lttng_session_list_tracker_ids(session, tracker_type);
 937         }
 938         case LTTNG_KERNEL_ABI_SESSION_METADATA_REGEN:
 939                 return lttng_session_metadata_regenerate(session);
 940         case LTTNG_KERNEL_ABI_SESSION_STATEDUMP:
 941                 return lttng_session_statedump(session);
 942         case LTTNG_KERNEL_ABI_SESSION_SET_NAME:
 943         {
 944                 struct lttng_kernel_abi_session_name name;
 945
 946                 if (copy_from_user(&name,
 947                                 (struct lttng_kernel_abi_session_name __user *) arg,
 948                                 sizeof(struct lttng_kernel_abi_session_name)))
 949                         return -EFAULT;
 950                 return lttng_abi_session_set_name(session, &name);
 951         }
 952         case LTTNG_KERNEL_ABI_SESSION_SET_CREATION_TIME:
 953         {
 954                 struct lttng_kernel_abi_session_creation_time time;
 955
 956                 if (copy_from_user(&time,
 957                                 (struct lttng_kernel_abi_session_creation_time __user *) arg,
 958                                 sizeof(struct lttng_kernel_abi_session_creation_time)))
 959                         return -EFAULT;
 960                 return lttng_abi_session_set_creation_time(session, &time);
 961         }
 962         default:
 963                 return -ENOIOCTLCMD;
 964         }
 965 }
 966
 967 /*
 968  * Called when the last file reference is dropped.
 969  *
 970  * Big fat note: channels and events are invariant for the whole session after
 971  * their creation. So this session destruction also destroys all channel and
 972  * event structures specific to this session (they are not destroyed when their
 973  * individual file is released).
 974  */
 975 static
 976 int lttng_session_release(struct inode *inode, struct file *file)
 977 {
 978         struct lttng_kernel_session *session = file->private_data;
 979
 980         if (session)
 981                 lttng_session_destroy(session);
 982         return 0;
 983 }
 984
 985 static const struct file_operations lttng_session_fops = {
 986         .owner = THIS_MODULE,
 987         .release = lttng_session_release,
 988         .unlocked_ioctl = lttng_session_ioctl,
 989 #ifdef CONFIG_COMPAT
 990         .compat_ioctl = lttng_session_ioctl,
 991 #endif
 992 };
 993
 994 /*
 995  * When encountering empty buffer, flush current sub-buffer if non-empty
 996  * and retry (if new data available to read after flush).
 997  */
 998 static
 999 ssize_t lttng_event_notifier_group_notif_read(struct file *filp, char __user *user_buf,
1000                 size_t count, loff_t *ppos)
1001 {
1002         struct lttng_event_notifier_group *event_notifier_group = filp->private_data;
1003         struct lttng_kernel_ring_buffer_channel *chan = event_notifier_group->chan;
1004         struct lttng_kernel_ring_buffer *buf = event_notifier_group->buf;
1005         ssize_t read_count = 0, len;
1006         size_t read_offset;
1007
1008         might_sleep();
1009         if (!lttng_access_ok(VERIFY_WRITE, user_buf, count))
1010                 return -EFAULT;
1011
1012         /* Finish copy of previous record */
1013         if (*ppos != 0) {
1014                 if (read_count < count) {
1015                         len = chan->iter.len_left;
1016                         read_offset = *ppos;
1017                         goto skip_get_next;
1018                 }
1019         }
1020
1021         while (read_count < count) {
1022                 size_t copy_len, space_left;
1023
1024                 len = lib_ring_buffer_get_next_record(chan, buf);
1025 len_test:
1026                 if (len < 0) {
1027                         /*
1028                          * Check if buffer is finalized (end of file).
1029                          */
1030                         if (len == -ENODATA) {
1031                                 /* A 0 read_count will tell about end of file */
1032                                 goto nodata;
1033                         }
1034                         if (filp->f_flags & O_NONBLOCK) {
1035                                 if (!read_count)
1036                                         read_count = -EAGAIN;
1037                                 goto nodata;
1038                         } else {
1039                                 int error;
1040
1041                                 /*
1042                                  * No data available at the moment, return what
1043                                  * we got.
1044                                  */
1045                                 if (read_count)
1046                                         goto nodata;
1047
1048                                 /*
1049                                  * Wait for returned len to be >= 0 or -ENODATA.
1050                                  */
1051                                 error = wait_event_interruptible(
1052                                           event_notifier_group->read_wait,
1053                                           ((len = lib_ring_buffer_get_next_record(
1054                                                   chan, buf)), len != -EAGAIN));
1055                                 CHAN_WARN_ON(chan, len == -EBUSY);
1056                                 if (error) {
1057                                         read_count = error;
1058                                         goto nodata;
1059                                 }
1060                                 CHAN_WARN_ON(chan, len < 0 && len != -ENODATA);
1061                                 goto len_test;
1062                         }
1063                 }
1064                 read_offset = buf->iter.read_offset;
1065 skip_get_next:
1066                 space_left = count - read_count;
1067                 if (len <= space_left) {
1068                         copy_len = len;
1069                         chan->iter.len_left = 0;
1070                         *ppos = 0;
1071                 } else {
1072                         copy_len = space_left;
1073                         chan->iter.len_left = len - copy_len;
1074                         *ppos = read_offset + copy_len;
1075                 }
1076                 if (__lib_ring_buffer_copy_to_user(&buf->backend, read_offset,
1077                                                &user_buf[read_count],
1078                                                copy_len)) {
1079                         /*
1080                          * Leave the len_left and ppos values at their current
1081                          * state, as we currently have a valid event to read.
1082                          */
1083                         return -EFAULT;
1084                 }
1085                 read_count += copy_len;
1086         }
1087         goto put_record;
1088
1089 nodata:
1090         *ppos = 0;
1091         chan->iter.len_left = 0;
1092
1093 put_record:
1094         lib_ring_buffer_put_current_record(buf);
1095         return read_count;
1096 }
1097
1098 /*
1099  * If the ring buffer is non empty (even just a partial subbuffer), return that
1100  * there is data available. Perform a ring buffer flush if we encounter a
1101  * non-empty ring buffer which does not have any consumeable subbuffer available.
1102  */
1103 static
1104 unsigned int lttng_event_notifier_group_notif_poll(struct file *filp,
1105                 poll_table *wait)
1106 {
1107         unsigned int mask = 0;
1108         struct lttng_event_notifier_group *event_notifier_group = filp->private_data;
1109         struct lttng_kernel_ring_buffer_channel *chan = event_notifier_group->chan;
1110         struct lttng_kernel_ring_buffer *buf = event_notifier_group->buf;
1111         const struct lttng_kernel_ring_buffer_config *config = &chan->backend.config;
1112         int finalized, disabled;
1113         unsigned long consumed, offset;
1114         size_t subbuffer_header_size = config->cb.subbuffer_header_size();
1115
1116         if (filp->f_mode & FMODE_READ) {
1117                 poll_wait_set_exclusive(wait);
1118                 poll_wait(filp, &event_notifier_group->read_wait, wait);
1119
1120                 finalized = lib_ring_buffer_is_finalized(config, buf);
1121                 disabled = lib_ring_buffer_channel_is_disabled(chan);
1122
1123                 /*
1124                  * lib_ring_buffer_is_finalized() contains a smp_rmb() ordering
1125                  * finalized load before offsets loads.
1126                  */
1127                 WARN_ON(atomic_long_read(&buf->active_readers) != 1);
1128 retry:
1129                 if (disabled)
1130                         return POLLERR;
1131
1132                 offset = lib_ring_buffer_get_offset(config, buf);
1133                 consumed = lib_ring_buffer_get_consumed(config, buf);
1134
1135                 /*
1136                  * If there is no buffer available to consume.
1137                  */
1138                 if (subbuf_trunc(offset, chan) - subbuf_trunc(consumed, chan) == 0) {
1139                         /*
1140                          * If there is a non-empty subbuffer, flush and try again.
1141                          */
1142                         if (subbuf_offset(offset, chan) > subbuffer_header_size) {
1143                                 lib_ring_buffer_switch_remote(buf);
1144                                 goto retry;
1145                         }
1146
1147                         if (finalized)
1148                                 return POLLHUP;
1149                         else {
1150                                 /*
1151                                  * The memory barriers
1152                                  * __wait_event()/wake_up_interruptible() take
1153                                  * care of "raw_spin_is_locked" memory ordering.
1154                                  */
1155                                 if (raw_spin_is_locked(&buf->raw_tick_nohz_spinlock))
1156                                         goto retry;
1157                                 else
1158                                         return 0;
1159                         }
1160                 } else {
1161                         if (subbuf_trunc(offset, chan) - subbuf_trunc(consumed, chan)
1162                                         >= chan->backend.buf_size)
1163                                 return POLLPRI | POLLRDBAND;
1164                         else
1165                                 return POLLIN | POLLRDNORM;
1166                 }
1167         }
1168
1169         return mask;
1170 }
1171
1172 /**
1173  *      lttng_event_notifier_group_notif_open - event_notifier ring buffer open file operation
1174  *      @inode: opened inode
1175  *      @file: opened file
1176  *
1177  *      Open implementation. Makes sure only one open instance of a buffer is
1178  *      done at a given moment.
1179  */
1180 static int lttng_event_notifier_group_notif_open(struct inode *inode, struct file *file)
1181 {
1182         struct lttng_event_notifier_group *event_notifier_group = inode->i_private;
1183         struct lttng_kernel_ring_buffer *buf = event_notifier_group->buf;
1184
1185         file->private_data = event_notifier_group;
1186         return lib_ring_buffer_open(inode, file, buf);
1187 }
1188
1189 /**
1190  *      lttng_event_notifier_group_notif_release - event_notifier ring buffer release file operation
1191  *      @inode: opened inode
1192  *      @file: opened file
1193  *
1194  *      Release implementation.
1195  */
1196 static int lttng_event_notifier_group_notif_release(struct inode *inode, struct file *file)
1197 {
1198         struct lttng_event_notifier_group *event_notifier_group = file->private_data;
1199         struct lttng_kernel_ring_buffer *buf = event_notifier_group->buf;
1200         int ret;
1201
1202         ret = lib_ring_buffer_release(inode, file, buf);
1203         if (ret)
1204                 return ret;
1205         fput(event_notifier_group->file);
1206         return 0;
1207 }
1208
1209 static const struct file_operations lttng_event_notifier_group_notif_fops = {
1210         .owner = THIS_MODULE,
1211         .open = lttng_event_notifier_group_notif_open,
1212         .release = lttng_event_notifier_group_notif_release,
1213         .read = lttng_event_notifier_group_notif_read,
1214         .poll = lttng_event_notifier_group_notif_poll,
1215 };
1216
1217 /**
1218  *      lttng_metadata_ring_buffer_poll - LTTng ring buffer poll file operation
1219  *      @filp: the file
1220  *      @wait: poll table
1221  *
1222  *      Handles the poll operations for the metadata channels.
1223  */
1224 static
1225 unsigned int lttng_metadata_ring_buffer_poll(struct file *filp,
1226                 poll_table *wait)
1227 {
1228         struct lttng_metadata_stream *stream = filp->private_data;
1229         struct lttng_kernel_ring_buffer *buf = stream->priv;
1230         int finalized;
1231         unsigned int mask = 0;
1232
1233         if (filp->f_mode & FMODE_READ) {
1234                 poll_wait_set_exclusive(wait);
1235                 poll_wait(filp, &stream->read_wait, wait);
1236
1237                 finalized = stream->finalized;
1238
1239                 /*
1240                  * lib_ring_buffer_is_finalized() contains a smp_rmb()
1241                  * ordering finalized load before offsets loads.
1242                  */
1243                 WARN_ON(atomic_long_read(&buf->active_readers) != 1);
1244
1245                 if (finalized)
1246                         mask |= POLLHUP;
1247
1248                 mutex_lock(&stream->metadata_cache->lock);
1249                 if (stream->metadata_cache->metadata_written >
1250                                 stream->metadata_out)
1251                         mask |= POLLIN;
1252                 mutex_unlock(&stream->metadata_cache->lock);
1253         }
1254
1255         return mask;
1256 }
1257
1258 static
1259 void lttng_metadata_ring_buffer_ioctl_put_next_subbuf(struct file *filp,
1260                 unsigned int cmd, unsigned long arg)
1261 {
1262         struct lttng_metadata_stream *stream = filp->private_data;
1263
1264         stream->metadata_out = stream->metadata_in;
1265 }
1266
1267 /*
1268  * Reset the counter of how much metadata has been consumed to 0. That way,
1269  * the consumer receives the content of the metadata cache unchanged. This is
1270  * different from the metadata_regenerate where the offset from epoch is
1271  * resampled, here we want the exact same content as the last time the metadata
1272  * was generated. This command is only possible if all the metadata written
1273  * in the cache has been output to the metadata stream to avoid corrupting the
1274  * metadata file.
1275  *
1276  * Return 0 on success, a negative value on error.
1277  */
1278 static
1279 int lttng_metadata_cache_dump(struct lttng_metadata_stream *stream)
1280 {
1281         int ret;
1282         struct lttng_metadata_cache *cache = stream->metadata_cache;
1283
1284         mutex_lock(&cache->lock);
1285         if (stream->metadata_out != cache->metadata_written) {
1286                 ret = -EBUSY;
1287                 goto end;
1288         }
1289         stream->metadata_out = 0;
1290         stream->metadata_in = 0;
1291         wake_up_interruptible(&stream->read_wait);
1292         ret = 0;
1293
1294 end:
1295         mutex_unlock(&cache->lock);
1296         return ret;
1297 }
1298
1299 static
1300 long lttng_metadata_ring_buffer_ioctl(struct file *filp,
1301                 unsigned int cmd, unsigned long arg)
1302 {
1303         int ret;
1304         struct lttng_metadata_stream *stream = filp->private_data;
1305         struct lttng_kernel_ring_buffer *buf = stream->priv;
1306         unsigned int rb_cmd;
1307         bool coherent;
1308
1309         if (cmd == LTTNG_KERNEL_ABI_RING_BUFFER_GET_NEXT_SUBBUF_METADATA_CHECK)
1310                 rb_cmd = LTTNG_KERNEL_ABI_RING_BUFFER_GET_NEXT_SUBBUF;
1311         else
1312                 rb_cmd = cmd;
1313
1314         switch (cmd) {
1315         case LTTNG_KERNEL_ABI_RING_BUFFER_GET_NEXT_SUBBUF:
1316         {
1317                 struct lttng_metadata_stream *stream = filp->private_data;
1318                 struct lttng_kernel_ring_buffer *buf = stream->priv;
1319                 struct lttng_kernel_ring_buffer_channel *chan = buf->backend.chan;
1320
1321                 ret = lttng_metadata_output_channel(stream, chan, NULL);
1322                 if (ret > 0) {
1323                         lib_ring_buffer_switch_slow(buf, SWITCH_ACTIVE);
1324                         ret = 0;
1325                 } else if (ret < 0)
1326                         goto err;
1327                 break;
1328         }
1329         case LTTNG_KERNEL_ABI_RING_BUFFER_GET_SUBBUF:
1330         {
1331                 /*
1332                  * Random access is not allowed for metadata channel.
1333                  */
1334                 return -ENOSYS;
1335         }
1336         case LTTNG_KERNEL_ABI_RING_BUFFER_FLUSH_EMPTY:
1337                 lttng_fallthrough;
1338         case LTTNG_KERNEL_ABI_RING_BUFFER_FLUSH:
1339         {
1340                 struct lttng_metadata_stream *stream = filp->private_data;
1341                 struct lttng_kernel_ring_buffer *buf = stream->priv;
1342                 struct lttng_kernel_ring_buffer_channel *chan = buf->backend.chan;
1343
1344                 /*
1345                  * Before doing the actual ring buffer flush, write up to one
1346                  * packet of metadata in the ring buffer.
1347                  */
1348                 ret = lttng_metadata_output_channel(stream, chan, NULL);
1349                 if (ret < 0)
1350                         goto err;
1351                 break;
1352         }
1353         case LTTNG_KERNEL_ABI_RING_BUFFER_GET_METADATA_VERSION:
1354         {
1355                 struct lttng_metadata_stream *stream = filp->private_data;
1356
1357                 return put_u64(stream->version, arg);
1358         }
1359         case LTTNG_KERNEL_ABI_RING_BUFFER_METADATA_CACHE_DUMP:
1360         {
1361                 struct lttng_metadata_stream *stream = filp->private_data;
1362
1363                 return lttng_metadata_cache_dump(stream);
1364         }
1365         case LTTNG_KERNEL_ABI_RING_BUFFER_GET_NEXT_SUBBUF_METADATA_CHECK:
1366         {
1367                 struct lttng_metadata_stream *stream = filp->private_data;
1368                 struct lttng_kernel_ring_buffer *buf = stream->priv;
1369                 struct lttng_kernel_ring_buffer_channel *chan = buf->backend.chan;
1370
1371                 ret = lttng_metadata_output_channel(stream, chan, &coherent);
1372                 if (ret > 0) {
1373                         lib_ring_buffer_switch_slow(buf, SWITCH_ACTIVE);
1374                         ret = 0;
1375                 } else if (ret < 0) {
1376                         goto err;
1377                 }
1378                 break;
1379         }
1380         default:
1381                 break;
1382         }
1383         /* PUT_SUBBUF is the one from lib ring buffer, unmodified. */
1384
1385         /* Performing lib ring buffer ioctl after our own. */
1386         ret = lib_ring_buffer_ioctl(filp, rb_cmd, arg, buf);
1387         if (ret < 0)
1388                 goto err;
1389
1390         switch (cmd) {
1391         case LTTNG_KERNEL_ABI_RING_BUFFER_PUT_NEXT_SUBBUF:
1392         {
1393                 lttng_metadata_ring_buffer_ioctl_put_next_subbuf(filp,
1394                                 cmd, arg);
1395                 break;
1396         }
1397         case LTTNG_KERNEL_ABI_RING_BUFFER_GET_NEXT_SUBBUF_METADATA_CHECK:
1398         {
1399                 return put_u32(coherent, arg);
1400         }
1401         default:
1402                 break;
1403         }
1404 err:
1405         return ret;
1406 }
1407
1408 #ifdef CONFIG_COMPAT
1409 static
1410 long lttng_metadata_ring_buffer_compat_ioctl(struct file *filp,
1411                 unsigned int cmd, unsigned long arg)
1412 {
1413         int ret;
1414         struct lttng_metadata_stream *stream = filp->private_data;
1415         struct lttng_kernel_ring_buffer *buf = stream->priv;
1416         unsigned int rb_cmd;
1417         bool coherent;
1418
1419         if (cmd == LTTNG_KERNEL_ABI_RING_BUFFER_GET_NEXT_SUBBUF_METADATA_CHECK)
1420                 rb_cmd = LTTNG_KERNEL_ABI_RING_BUFFER_GET_NEXT_SUBBUF;
1421         else
1422                 rb_cmd = cmd;
1423
1424         switch (cmd) {
1425         case LTTNG_KERNEL_ABI_RING_BUFFER_GET_NEXT_SUBBUF:
1426         {
1427                 struct lttng_metadata_stream *stream = filp->private_data;
1428                 struct lttng_kernel_ring_buffer *buf = stream->priv;
1429                 struct lttng_kernel_ring_buffer_channel *chan = buf->backend.chan;
1430
1431                 ret = lttng_metadata_output_channel(stream, chan, NULL);
1432                 if (ret > 0) {
1433                         lib_ring_buffer_switch_slow(buf, SWITCH_ACTIVE);
1434                         ret = 0;
1435                 } else if (ret < 0)
1436                         goto err;
1437                 break;
1438         }
1439         case LTTNG_KERNEL_ABI_RING_BUFFER_GET_SUBBUF:
1440         {
1441                 /*
1442                  * Random access is not allowed for metadata channel.
1443                  */
1444                 return -ENOSYS;
1445         }
1446         case LTTNG_KERNEL_ABI_RING_BUFFER_FLUSH_EMPTY:
1447                 lttng_fallthrough;
1448         case LTTNG_KERNEL_ABI_RING_BUFFER_FLUSH:
1449         {
1450                 struct lttng_metadata_stream *stream = filp->private_data;
1451                 struct lttng_kernel_ring_buffer *buf = stream->priv;
1452                 struct lttng_kernel_ring_buffer_channel *chan = buf->backend.chan;
1453
1454                 /*
1455                  * Before doing the actual ring buffer flush, write up to one
1456                  * packet of metadata in the ring buffer.
1457                  */
1458                 ret = lttng_metadata_output_channel(stream, chan, NULL);
1459                 if (ret < 0)
1460                         goto err;
1461                 break;
1462         }
1463         case LTTNG_KERNEL_ABI_RING_BUFFER_GET_METADATA_VERSION:
1464         {
1465                 struct lttng_metadata_stream *stream = filp->private_data;
1466
1467                 return put_u64(stream->version, arg);
1468         }
1469         case LTTNG_KERNEL_ABI_RING_BUFFER_METADATA_CACHE_DUMP:
1470         {
1471                 struct lttng_metadata_stream *stream = filp->private_data;
1472
1473                 return lttng_metadata_cache_dump(stream);
1474         }
1475         case LTTNG_KERNEL_ABI_RING_BUFFER_GET_NEXT_SUBBUF_METADATA_CHECK:
1476         {
1477                 struct lttng_metadata_stream *stream = filp->private_data;
1478                 struct lttng_kernel_ring_buffer *buf = stream->priv;
1479                 struct lttng_kernel_ring_buffer_channel *chan = buf->backend.chan;
1480
1481                 ret = lttng_metadata_output_channel(stream, chan, &coherent);
1482                 if (ret > 0) {
1483                         lib_ring_buffer_switch_slow(buf, SWITCH_ACTIVE);
1484                         ret = 0;
1485                 } else if (ret < 0) {
1486                         goto err;
1487                 }
1488                 break;
1489         }
1490         default:
1491                 break;
1492         }
1493         /* PUT_SUBBUF is the one from lib ring buffer, unmodified. */
1494
1495         /* Performing lib ring buffer ioctl after our own. */
1496         ret = lib_ring_buffer_compat_ioctl(filp, rb_cmd, arg, buf);
1497         if (ret < 0)
1498                 goto err;
1499
1500         switch (cmd) {
1501         case LTTNG_KERNEL_ABI_RING_BUFFER_PUT_NEXT_SUBBUF:
1502         {
1503                 lttng_metadata_ring_buffer_ioctl_put_next_subbuf(filp,
1504                                 cmd, arg);
1505                 break;
1506         }
1507         case LTTNG_KERNEL_ABI_RING_BUFFER_GET_NEXT_SUBBUF_METADATA_CHECK:
1508         {
1509                 return put_u32(coherent, arg);
1510         }
1511         default:
1512                 break;
1513         }
1514 err:
1515         return ret;
1516 }
1517 #endif
1518
1519 /*
1520  * This is not used by anonymous file descriptors. This code is left
1521  * there if we ever want to implement an inode with open() operation.
1522  */
1523 static
1524 int lttng_metadata_ring_buffer_open(struct inode *inode, struct file *file)
1525 {
1526         struct lttng_metadata_stream *stream = inode->i_private;
1527         struct lttng_kernel_ring_buffer *buf = stream->priv;
1528
1529         file->private_data = buf;
1530         /*
1531          * Since life-time of metadata cache differs from that of
1532          * session, we need to keep our own reference on the transport.
1533          */
1534         if (!try_module_get(stream->transport->owner)) {
1535                 printk(KERN_WARNING "LTTng: Can't lock transport module.\n");
1536                 return -EBUSY;
1537         }
1538         return lib_ring_buffer_open(inode, file, buf);
1539 }
1540
1541 static
1542 int lttng_metadata_ring_buffer_release(struct inode *inode, struct file *file)
1543 {
1544         struct lttng_metadata_stream *stream = file->private_data;
1545         struct lttng_kernel_ring_buffer *buf = stream->priv;
1546
1547         mutex_lock(&stream->metadata_cache->lock);
1548         list_del(&stream->list);
1549         mutex_unlock(&stream->metadata_cache->lock);
1550         kref_put(&stream->metadata_cache->refcount, metadata_cache_destroy);
1551         module_put(stream->transport->owner);
1552         kfree(stream);
1553         return lib_ring_buffer_release(inode, file, buf);
1554 }
1555
1556 static
1557 ssize_t lttng_metadata_ring_buffer_splice_read(struct file *in, loff_t *ppos,
1558                 struct pipe_inode_info *pipe, size_t len,
1559                 unsigned int flags)
1560 {
1561         struct lttng_metadata_stream *stream = in->private_data;
1562         struct lttng_kernel_ring_buffer *buf = stream->priv;
1563
1564         return lib_ring_buffer_splice_read(in, ppos, pipe, len,
1565                         flags, buf);
1566 }
1567
1568 static
1569 int lttng_metadata_ring_buffer_mmap(struct file *filp,
1570                 struct vm_area_struct *vma)
1571 {
1572         struct lttng_metadata_stream *stream = filp->private_data;
1573         struct lttng_kernel_ring_buffer *buf = stream->priv;
1574
1575         return lib_ring_buffer_mmap(filp, vma, buf);
1576 }
1577
1578 static
1579 const struct file_operations lttng_metadata_ring_buffer_file_operations = {
1580         .owner = THIS_MODULE,
1581         .open = lttng_metadata_ring_buffer_open,
1582         .release = lttng_metadata_ring_buffer_release,
1583         .poll = lttng_metadata_ring_buffer_poll,
1584         .splice_read = lttng_metadata_ring_buffer_splice_read,
1585         .mmap = lttng_metadata_ring_buffer_mmap,
1586         .unlocked_ioctl = lttng_metadata_ring_buffer_ioctl,
1587         .llseek = vfs_lib_ring_buffer_no_llseek,
1588 #ifdef CONFIG_COMPAT
1589         .compat_ioctl = lttng_metadata_ring_buffer_compat_ioctl,
1590 #endif
1591 };
1592
1593 static
1594 int lttng_abi_create_stream_fd(struct file *channel_file, void *stream_priv,
1595                 const struct file_operations *fops, const char *name)
1596 {
1597         int stream_fd, ret;
1598         struct file *stream_file;
1599
1600         stream_fd = lttng_get_unused_fd();
1601         if (stream_fd < 0) {
1602                 ret = stream_fd;
1603                 goto fd_error;
1604         }
1605         stream_file = anon_inode_getfile(name, fops, stream_priv, O_RDWR);
1606         if (IS_ERR(stream_file)) {
1607                 ret = PTR_ERR(stream_file);
1608                 goto file_error;
1609         }
1610         /*
1611          * OPEN_FMODE, called within anon_inode_getfile/alloc_file, don't honor
1612          * FMODE_LSEEK, FMODE_PREAD nor FMODE_PWRITE. We need to read from this
1613          * file descriptor, so we set FMODE_PREAD here.
1614          */
1615         stream_file->f_mode |= FMODE_PREAD;
1616         fd_install(stream_fd, stream_file);
1617         /*
1618          * The stream holds a reference to the channel within the generic ring
1619          * buffer library, so no need to hold a refcount on the channel and
1620          * session files here.
1621          */
1622         return stream_fd;
1623
1624 file_error:
1625         put_unused_fd(stream_fd);
1626 fd_error:
1627         return ret;
1628 }
1629
1630 static
1631 int lttng_abi_open_stream(struct file *channel_file)
1632 {
1633         struct lttng_kernel_channel_buffer *channel = channel_file->private_data;
1634         struct lttng_kernel_ring_buffer *buf;
1635         int ret;
1636         void *stream_priv;
1637
1638         buf = channel->ops->priv->buffer_read_open(channel->priv->rb_chan);
1639         if (!buf)
1640                 return -ENOENT;
1641
1642         stream_priv = buf;
1643         ret = lttng_abi_create_stream_fd(channel_file, stream_priv,
1644                         &lttng_stream_ring_buffer_file_operations,
1645                         "[lttng_stream]");
1646         if (ret < 0)
1647                 goto fd_error;
1648
1649         return ret;
1650
1651 fd_error:
1652         channel->ops->priv->buffer_read_close(buf);
1653         return ret;
1654 }
1655
1656 static
1657 int lttng_abi_open_metadata_stream(struct file *channel_file)
1658 {
1659         struct lttng_kernel_channel_buffer *channel = channel_file->private_data;
1660         struct lttng_kernel_session *session = channel->parent.session;
1661         struct lttng_kernel_ring_buffer *buf;
1662         int ret;
1663         struct lttng_metadata_stream *metadata_stream;
1664         void *stream_priv;
1665
1666         buf = channel->ops->priv->buffer_read_open(channel->priv->rb_chan);
1667         if (!buf)
1668                 return -ENOENT;
1669
1670         metadata_stream = kzalloc(sizeof(struct lttng_metadata_stream),
1671                         GFP_KERNEL);
1672         if (!metadata_stream) {
1673                 ret = -ENOMEM;
1674                 goto nomem;
1675         }
1676         metadata_stream->metadata_cache = session->priv->metadata_cache;
1677         init_waitqueue_head(&metadata_stream->read_wait);
1678         metadata_stream->priv = buf;
1679         stream_priv = metadata_stream;
1680         metadata_stream->transport = channel->priv->transport;
1681         /* Initial state is an empty metadata, considered as incoherent. */
1682         metadata_stream->coherent = false;
1683
1684         /*
1685          * Since life-time of metadata cache differs from that of
1686          * session, we need to keep our own reference on the transport.
1687          */
1688         if (!try_module_get(metadata_stream->transport->owner)) {
1689                 printk(KERN_WARNING "LTTng: Can't lock transport module.\n");
1690                 ret = -EINVAL;
1691                 goto notransport;
1692         }
1693
1694         if (!lttng_kref_get(&session->priv->metadata_cache->refcount)) {
1695                 ret = -EOVERFLOW;
1696                 goto kref_error;
1697         }
1698
1699         ret = lttng_abi_create_stream_fd(channel_file, stream_priv,
1700                         &lttng_metadata_ring_buffer_file_operations,
1701                         "[lttng_metadata_stream]");
1702         if (ret < 0)
1703                 goto fd_error;
1704
1705         mutex_lock(&session->priv->metadata_cache->lock);
1706         list_add(&metadata_stream->list,
1707                 &session->priv->metadata_cache->metadata_stream);
1708         mutex_unlock(&session->priv->metadata_cache->lock);
1709         return ret;
1710
1711 fd_error:
1712         kref_put(&session->priv->metadata_cache->refcount, metadata_cache_destroy);
1713 kref_error:
1714         module_put(metadata_stream->transport->owner);
1715 notransport:
1716         kfree(metadata_stream);
1717 nomem:
1718         channel->ops->priv->buffer_read_close(buf);
1719         return ret;
1720 }
1721
1722 static
1723 int lttng_abi_open_event_notifier_group_stream(struct file *notif_file)
1724 {
1725         struct lttng_event_notifier_group *event_notifier_group = notif_file->private_data;
1726         struct lttng_kernel_ring_buffer_channel *chan = event_notifier_group->chan;
1727         struct lttng_kernel_ring_buffer *buf;
1728         int ret;
1729         void *stream_priv;
1730
1731         buf = event_notifier_group->ops->priv->buffer_read_open(chan);
1732         if (!buf)
1733                 return -ENOENT;
1734
1735         /* The event_notifier notification fd holds a reference on the event_notifier group */
1736         if (!atomic_long_add_unless(&notif_file->f_count, 1, LONG_MAX)) {
1737                 ret = -EOVERFLOW;
1738                 goto refcount_error;
1739         }
1740         event_notifier_group->buf = buf;
1741         stream_priv = event_notifier_group;
1742         ret = lttng_abi_create_stream_fd(notif_file, stream_priv,
1743                         &lttng_event_notifier_group_notif_fops,
1744                         "[lttng_event_notifier_stream]");
1745         if (ret < 0)
1746                 goto fd_error;
1747
1748         return ret;
1749
1750 fd_error:
1751         atomic_long_dec(&notif_file->f_count);
1752 refcount_error:
1753         event_notifier_group->ops->priv->buffer_read_close(buf);
1754         return ret;
1755 }
1756
1757 static
1758 int lttng_abi_validate_event_param(struct lttng_kernel_abi_event *event_param)
1759 {
1760         /* Limit ABI to implemented features. */
1761         switch (event_param->instrumentation) {
1762         case LTTNG_KERNEL_ABI_SYSCALL:
1763                 switch (event_param->u.syscall.entryexit) {
1764                 case LTTNG_KERNEL_ABI_SYSCALL_ENTRY:
1765                         lttng_fallthrough;
1766                 case LTTNG_KERNEL_ABI_SYSCALL_EXIT:
1767                         lttng_fallthrough;
1768                 case LTTNG_KERNEL_ABI_SYSCALL_ENTRYEXIT:
1769                         break;
1770                 default:
1771                         return -EINVAL;
1772                 }
1773                 switch (event_param->u.syscall.abi) {
1774                 case LTTNG_KERNEL_ABI_SYSCALL_ABI_ALL:
1775                         break;
1776                 default:
1777                         return -EINVAL;
1778                 }
1779                 switch (event_param->u.syscall.match) {
1780                 case LTTNG_KERNEL_ABI_SYSCALL_MATCH_NAME:
1781                         break;
1782                 default:
1783                         return -EINVAL;
1784                 }
1785                 break;
1786
1787         case LTTNG_KERNEL_ABI_KRETPROBE:
1788                 switch (event_param->u.kretprobe.entryexit) {
1789                 case LTTNG_KERNEL_ABI_SYSCALL_ENTRYEXIT:
1790                         break;
1791                 case LTTNG_KERNEL_ABI_SYSCALL_ENTRY:
1792                         lttng_fallthrough;
1793                 case LTTNG_KERNEL_ABI_SYSCALL_EXIT:
1794                         lttng_fallthrough;
1795                 default:
1796                         return -EINVAL;
1797                 }
1798                 break;
1799
1800         case LTTNG_KERNEL_ABI_TRACEPOINT:
1801                 lttng_fallthrough;
1802         case LTTNG_KERNEL_ABI_KPROBE:
1803                 lttng_fallthrough;
1804         case LTTNG_KERNEL_ABI_UPROBE:
1805                 break;
1806
1807         case LTTNG_KERNEL_ABI_FUNCTION:
1808                 lttng_fallthrough;
1809         case LTTNG_KERNEL_ABI_NOOP:
1810                 lttng_fallthrough;
1811         default:
1812                 return -EINVAL;
1813         }
1814         return 0;
1815 }
1816
1817 static
1818 int lttng_abi_create_event(struct file *channel_file,
1819                            struct lttng_kernel_abi_event *event_param)
1820 {
1821         const struct file_operations *fops;
1822         struct lttng_kernel_channel_buffer *channel = channel_file->private_data;
1823         int event_fd, ret;
1824         struct file *event_file;
1825         void *priv;
1826
1827         event_param->name[LTTNG_KERNEL_ABI_SYM_NAME_LEN - 1] = '\0';
1828         switch (event_param->instrumentation) {
1829         case LTTNG_KERNEL_ABI_KRETPROBE:
1830                 event_param->u.kretprobe.symbol_name[LTTNG_KERNEL_ABI_SYM_NAME_LEN - 1] = '\0';
1831                 break;
1832         case LTTNG_KERNEL_ABI_KPROBE:
1833                 event_param->u.kprobe.symbol_name[LTTNG_KERNEL_ABI_SYM_NAME_LEN - 1] = '\0';
1834                 break;
1835         case LTTNG_KERNEL_ABI_FUNCTION:
1836                 WARN_ON_ONCE(1);
1837                 /* Not implemented. */
1838                 break;
1839         default:
1840                 break;
1841         }
1842
1843         switch (event_param->instrumentation) {
1844         case LTTNG_KERNEL_ABI_TRACEPOINT:
1845                 lttng_fallthrough;
1846         case LTTNG_KERNEL_ABI_SYSCALL:
1847                 fops = &lttng_event_recorder_enabler_fops;
1848                 break;
1849         case LTTNG_KERNEL_ABI_KPROBE:
1850                 lttng_fallthrough;
1851         case LTTNG_KERNEL_ABI_KRETPROBE:
1852                 lttng_fallthrough;
1853         case LTTNG_KERNEL_ABI_UPROBE:
1854                 fops = &lttng_event_recorder_event_fops;
1855                 break;
1856
1857         case LTTNG_KERNEL_ABI_FUNCTION:
1858                 lttng_fallthrough;
1859         case LTTNG_KERNEL_ABI_NOOP:
1860                 lttng_fallthrough;
1861         default:
1862                 return -EINVAL;
1863         }
1864
1865         event_fd = lttng_get_unused_fd();
1866         if (event_fd < 0) {
1867                 ret = event_fd;
1868                 goto fd_error;
1869         }
1870         event_file = anon_inode_getfile("[lttng_event]",
1871                                         fops, NULL, O_RDWR);
1872         if (IS_ERR(event_file)) {
1873                 ret = PTR_ERR(event_file);
1874                 goto file_error;
1875         }
1876         /* The event holds a reference on the channel */
1877         if (!atomic_long_add_unless(&channel_file->f_count, 1, LONG_MAX)) {
1878                 ret = -EOVERFLOW;
1879                 goto refcount_error;
1880         }
1881         ret = lttng_abi_validate_event_param(event_param);
1882         if (ret)
1883                 goto event_error;
1884
1885         switch (event_param->instrumentation) {
1886         case LTTNG_KERNEL_ABI_TRACEPOINT:
1887                 lttng_fallthrough;
1888         case LTTNG_KERNEL_ABI_SYSCALL:
1889         {
1890                 struct lttng_event_recorder_enabler *event_enabler;
1891
1892                 if (strutils_is_star_glob_pattern(event_param->name)) {
1893                         /*
1894                          * If the event name is a star globbing pattern,
1895                          * we create the special star globbing enabler.
1896                          */
1897                         event_enabler = lttng_event_recorder_enabler_create(LTTNG_ENABLER_FORMAT_STAR_GLOB,
1898                                 event_param, channel);
1899                 } else {
1900                         event_enabler = lttng_event_recorder_enabler_create(LTTNG_ENABLER_FORMAT_NAME,
1901                                 event_param, channel);
1902                 }
1903                 if (event_enabler)
1904                         lttng_event_enabler_session_add(channel->parent.session, event_enabler);
1905                 priv = event_enabler;
1906                 break;
1907         }
1908
1909         case LTTNG_KERNEL_ABI_KPROBE:
1910                 lttng_fallthrough;
1911         case LTTNG_KERNEL_ABI_KRETPROBE:
1912                 lttng_fallthrough;
1913         case LTTNG_KERNEL_ABI_UPROBE:
1914         {
1915                 struct lttng_kernel_event_recorder *event;
1916                 struct lttng_event_recorder_enabler *event_enabler;
1917
1918                 event_enabler = lttng_event_recorder_enabler_create(LTTNG_ENABLER_FORMAT_NAME,
1919                                 event_param, channel);
1920                 if (!event_enabler) {
1921                         ret = -ENOMEM;
1922                         goto event_error;
1923                 }
1924                 /*
1925                  * We tolerate no failure path after event creation. It
1926                  * will stay invariant for the rest of the session.
1927                  */
1928                 event = lttng_kernel_event_recorder_create(event_enabler, NULL);
1929                 WARN_ON_ONCE(!event);
1930                 lttng_event_enabler_destroy(event_enabler);
1931                 if (IS_ERR(event)) {
1932                         ret = PTR_ERR(event);
1933                         goto event_error;
1934                 }
1935                 priv = event;
1936                 break;
1937         }
1938
1939         case LTTNG_KERNEL_ABI_FUNCTION:
1940                 lttng_fallthrough;
1941         case LTTNG_KERNEL_ABI_NOOP:
1942                 lttng_fallthrough;
1943         default:
1944                 ret = -EINVAL;
1945                 goto event_error;
1946         }
1947         event_file->private_data = priv;
1948         fd_install(event_fd, event_file);
1949         return event_fd;
1950
1951 event_error:
1952         atomic_long_dec(&channel_file->f_count);
1953 refcount_error:
1954         fput(event_file);
1955 file_error:
1956         put_unused_fd(event_fd);
1957 fd_error:
1958         return ret;
1959 }
1960
1961 static
1962 long lttng_event_notifier_event_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1963 {
1964         struct lttng_kernel_event_notifier *event_notifier = file->private_data;
1965
1966         switch (cmd) {
1967         case LTTNG_KERNEL_ABI_ENABLE:
1968                 return lttng_event_enable(&event_notifier->parent);
1969         case LTTNG_KERNEL_ABI_DISABLE:
1970                 return lttng_event_disable(&event_notifier->parent);
1971         case LTTNG_KERNEL_ABI_FILTER:
1972                 return -EINVAL;
1973         case LTTNG_KERNEL_ABI_CAPTURE:
1974                 return -EINVAL;
1975         case LTTNG_KERNEL_ABI_ADD_CALLSITE:
1976                 return lttng_event_add_callsite(&event_notifier->parent,
1977                         (struct lttng_kernel_abi_event_callsite __user *) arg);
1978         default:
1979                 return -ENOIOCTLCMD;
1980         }
1981 }
1982
1983 static
1984 long lttng_event_notifier_enabler_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1985 {
1986         struct lttng_event_notifier_enabler *event_notifier_enabler = file->private_data;
1987
1988         switch (cmd) {
1989         case LTTNG_KERNEL_ABI_ENABLE:
1990                 return lttng_event_enabler_enable(&event_notifier_enabler->parent);
1991         case LTTNG_KERNEL_ABI_DISABLE:
1992                 return lttng_event_enabler_disable(&event_notifier_enabler->parent);
1993         case LTTNG_KERNEL_ABI_FILTER:
1994                 return lttng_event_enabler_attach_filter_bytecode(&event_notifier_enabler->parent,
1995                         (struct lttng_kernel_abi_filter_bytecode __user *) arg);
1996         case LTTNG_KERNEL_ABI_CAPTURE:
1997                 return lttng_event_notifier_enabler_attach_capture_bytecode(
1998                         event_notifier_enabler,
1999                         (struct lttng_kernel_abi_capture_bytecode __user *) arg);
2000         case LTTNG_KERNEL_ABI_ADD_CALLSITE:
2001                 return -EINVAL;
2002         default:
2003                 return -ENOIOCTLCMD;
2004         }
2005 }
2006
2007 static
2008 int lttng_event_notifier_event_release(struct inode *inode, struct file *file)
2009 {
2010         struct lttng_kernel_event_notifier *event_notifier = file->private_data;
2011
2012         if (event_notifier)
2013                 fput(event_notifier->priv->group->file);
2014         return 0;
2015 }
2016
2017 static
2018 int lttng_event_notifier_enabler_release(struct inode *inode, struct file *file)
2019 {
2020         struct lttng_event_notifier_enabler *event_notifier_enabler = file->private_data;
2021
2022         if (event_notifier_enabler)
2023                 fput(event_notifier_enabler->group->file);
2024         return 0;
2025 }
2026
2027 static const struct file_operations lttng_event_notifier_event_fops = {
2028         .owner = THIS_MODULE,
2029         .release = lttng_event_notifier_event_release,
2030         .unlocked_ioctl = lttng_event_notifier_event_ioctl,
2031 #ifdef CONFIG_COMPAT
2032         .compat_ioctl = lttng_event_notifier_event_ioctl,
2033 #endif
2034 };
2035
2036 static const struct file_operations lttng_event_notifier_enabler_fops = {
2037         .owner = THIS_MODULE,
2038         .release = lttng_event_notifier_enabler_release,
2039         .unlocked_ioctl = lttng_event_notifier_enabler_ioctl,
2040 #ifdef CONFIG_COMPAT
2041         .compat_ioctl = lttng_event_notifier_enabler_ioctl,
2042 #endif
2043 };
2044
2045 static
2046 int lttng_abi_create_event_notifier(struct file *event_notifier_group_file,
2047                 struct lttng_kernel_abi_event_notifier *event_notifier_param)
2048 {
2049         struct lttng_event_notifier_group *event_notifier_group =
2050                         event_notifier_group_file->private_data;
2051         const struct file_operations *fops;
2052         int event_notifier_fd, ret;
2053         struct file *event_notifier_file;
2054         void *priv;
2055
2056         switch (event_notifier_param->event.instrumentation) {
2057         case LTTNG_KERNEL_ABI_TRACEPOINT:
2058         case LTTNG_KERNEL_ABI_UPROBE:
2059                 break;
2060         case LTTNG_KERNEL_ABI_KPROBE:
2061                 event_notifier_param->event.u.kprobe.symbol_name[LTTNG_KERNEL_ABI_SYM_NAME_LEN - 1] = '\0';
2062                 break;
2063         case LTTNG_KERNEL_ABI_SYSCALL:
2064                 break;
2065         case LTTNG_KERNEL_ABI_KRETPROBE:
2066                 /* Placing an event notifier on kretprobe is not supported. */
2067         case LTTNG_KERNEL_ABI_FUNCTION:
2068         case LTTNG_KERNEL_ABI_NOOP:
2069         default:
2070                 ret = -EINVAL;
2071                 goto inval_instr;
2072         }
2073
2074         switch (event_notifier_param->event.instrumentation) {
2075         case LTTNG_KERNEL_ABI_TRACEPOINT:
2076                 lttng_fallthrough;
2077         case LTTNG_KERNEL_ABI_SYSCALL:
2078                 fops = &lttng_event_notifier_enabler_fops;
2079                 break;
2080         case LTTNG_KERNEL_ABI_KPROBE:
2081                 lttng_fallthrough;
2082         case LTTNG_KERNEL_ABI_KRETPROBE:
2083                 lttng_fallthrough;
2084         case LTTNG_KERNEL_ABI_UPROBE:
2085                 fops = &lttng_event_notifier_event_fops;
2086                 break;
2087
2088         case LTTNG_KERNEL_ABI_FUNCTION:
2089                 lttng_fallthrough;
2090         case LTTNG_KERNEL_ABI_NOOP:
2091                 lttng_fallthrough;
2092         default:
2093                 ret = -EINVAL;
2094                 goto inval_instr;
2095         }
2096
2097         event_notifier_param->event.name[LTTNG_KERNEL_ABI_SYM_NAME_LEN - 1] = '\0';
2098
2099         event_notifier_fd = lttng_get_unused_fd();
2100         if (event_notifier_fd < 0) {
2101                 ret = event_notifier_fd;
2102                 goto fd_error;
2103         }
2104
2105         event_notifier_file = anon_inode_getfile("[lttng_event_notifier]",
2106                                         fops, NULL, O_RDWR);
2107         if (IS_ERR(event_notifier_file)) {
2108                 ret = PTR_ERR(event_notifier_file);
2109                 goto file_error;
2110         }
2111
2112         /* The event notifier holds a reference on the event notifier group. */
2113         if (!atomic_long_add_unless(&event_notifier_group_file->f_count, 1, LONG_MAX)) {
2114                 ret = -EOVERFLOW;
2115                 goto refcount_error;
2116         }
2117
2118         ret = lttng_abi_validate_event_param(&event_notifier_param->event);
2119         if (ret)
2120                 goto event_notifier_error;
2121
2122         switch (event_notifier_param->event.instrumentation) {
2123         case LTTNG_KERNEL_ABI_TRACEPOINT:
2124                 lttng_fallthrough;
2125         case LTTNG_KERNEL_ABI_SYSCALL:
2126         {
2127                 struct lttng_event_notifier_enabler *enabler;
2128
2129                 if (strutils_is_star_glob_pattern(event_notifier_param->event.name)) {
2130                         /*
2131                          * If the event name is a star globbing pattern,
2132                          * we create the special star globbing enabler.
2133                          */
2134                         enabler = lttng_event_notifier_enabler_create(
2135                                         event_notifier_group,
2136                                         LTTNG_ENABLER_FORMAT_STAR_GLOB,
2137                                         event_notifier_param);
2138                 } else {
2139                         enabler = lttng_event_notifier_enabler_create(
2140                                         event_notifier_group,
2141                                         LTTNG_ENABLER_FORMAT_NAME,
2142                                         event_notifier_param);
2143                 }
2144                 priv = enabler;
2145                 break;
2146         }
2147
2148         case LTTNG_KERNEL_ABI_KPROBE:
2149                 lttng_fallthrough;
2150         case LTTNG_KERNEL_ABI_KRETPROBE:
2151                 lttng_fallthrough;
2152         case LTTNG_KERNEL_ABI_UPROBE:
2153         {
2154                 struct lttng_kernel_event_notifier *event_notifier;
2155
2156                 /*
2157                  * We tolerate no failure path after event notifier creation.
2158                  * It will stay invariant for the rest of the session.
2159                  */
2160                 event_notifier = lttng_event_notifier_create(NULL,
2161                                 event_notifier_param->event.token,
2162                                 event_notifier_param->error_counter_index,
2163                                 event_notifier_group,
2164                                 event_notifier_param,
2165                                 event_notifier_param->event.instrumentation);
2166                 WARN_ON_ONCE(!event_notifier);
2167                 if (IS_ERR(event_notifier)) {
2168                         ret = PTR_ERR(event_notifier);
2169                         goto event_notifier_error;
2170                 }
2171                 priv = event_notifier;
2172                 break;
2173         }
2174
2175         case LTTNG_KERNEL_ABI_FUNCTION:
2176                 lttng_fallthrough;
2177         case LTTNG_KERNEL_ABI_NOOP:
2178                 lttng_fallthrough;
2179         default:
2180                 ret = -EINVAL;
2181                 goto event_notifier_error;
2182         }
2183         event_notifier_file->private_data = priv;
2184         fd_install(event_notifier_fd, event_notifier_file);
2185         return event_notifier_fd;
2186
2187 event_notifier_error:
2188         atomic_long_dec(&event_notifier_group_file->f_count);
2189 refcount_error:
2190         fput(event_notifier_file);
2191 file_error:
2192         put_unused_fd(event_notifier_fd);
2193 fd_error:
2194 inval_instr:
2195         return ret;
2196 }
2197
2198 static
2199 long lttng_abi_event_notifier_group_create_error_counter(
2200                 struct file *event_notifier_group_file,
2201                 const struct lttng_kernel_abi_counter_conf *error_counter_conf)
2202 {
2203         int counter_fd, ret;
2204         char *counter_transport_name;
2205         size_t counter_len;
2206         struct lttng_counter *counter = NULL;
2207         struct file *counter_file;
2208         struct lttng_event_notifier_group *event_notifier_group =
2209                         (struct lttng_event_notifier_group *) event_notifier_group_file->private_data;
2210
2211         if (error_counter_conf->arithmetic != LTTNG_KERNEL_ABI_COUNTER_ARITHMETIC_MODULAR) {
2212                 printk(KERN_ERR "LTTng: event_notifier: Error counter of the wrong arithmetic type.\n");
2213                 return -EINVAL;
2214         }
2215
2216         if (error_counter_conf->number_dimensions != 1) {
2217                 printk(KERN_ERR "LTTng: event_notifier: Error counter has more than one dimension.\n");
2218                 return -EINVAL;
2219         }
2220
2221         switch (error_counter_conf->bitness) {
2222         case LTTNG_KERNEL_ABI_COUNTER_BITNESS_64:
2223                 counter_transport_name = "counter-per-cpu-64-modular";
2224                 break;
2225         case LTTNG_KERNEL_ABI_COUNTER_BITNESS_32:
2226                 counter_transport_name = "counter-per-cpu-32-modular";
2227                 break;
2228         default:
2229                 return -EINVAL;
2230         }
2231
2232         /*
2233          * Lock sessions to provide mutual exclusion against concurrent
2234          * modification of event_notifier group, which would result in
2235          * overwriting the error counter if set concurrently.
2236          */
2237         lttng_lock_sessions();
2238
2239         if (event_notifier_group->error_counter) {
2240                 printk(KERN_ERR "Error counter already created in event_notifier group\n");
2241                 ret = -EBUSY;
2242                 goto fd_error;
2243         }
2244
2245         counter_fd = lttng_get_unused_fd();
2246         if (counter_fd < 0) {
2247                 ret = counter_fd;
2248                 goto fd_error;
2249         }
2250
2251         counter_file = anon_inode_getfile("[lttng_counter]",
2252                                        &lttng_counter_fops,
2253                                        NULL, O_RDONLY);
2254         if (IS_ERR(counter_file)) {
2255                 ret = PTR_ERR(counter_file);
2256                 goto file_error;
2257         }
2258
2259         counter_len = error_counter_conf->dimensions[0].size;
2260
2261         if (!atomic_long_add_unless(&event_notifier_group_file->f_count, 1, LONG_MAX)) {
2262                 ret = -EOVERFLOW;
2263                 goto refcount_error;
2264         }
2265
2266         counter = lttng_kernel_counter_create(counter_transport_name,
2267                         1, &counter_len);
2268         if (!counter) {
2269                 ret = -EINVAL;
2270                 goto counter_error;
2271         }
2272
2273         event_notifier_group->error_counter_len = counter_len;
2274         /*
2275          * store-release to publish error counter matches load-acquire
2276          * in record_error. Ensures the counter is created and the
2277          * error_counter_len is set before they are used.
2278          */
2279         lttng_smp_store_release(&event_notifier_group->error_counter, counter);
2280
2281         counter->file = counter_file;
2282         counter->owner = event_notifier_group->file;
2283         counter_file->private_data = counter;
2284         /* Ownership transferred. */
2285         counter = NULL;
2286
2287         fd_install(counter_fd, counter_file);
2288         lttng_unlock_sessions();
2289
2290         return counter_fd;
2291
2292 counter_error:
2293         atomic_long_dec(&event_notifier_group_file->f_count);
2294 refcount_error:
2295         fput(counter_file);
2296 file_error:
2297         put_unused_fd(counter_fd);
2298 fd_error:
2299         lttng_unlock_sessions();
2300         return ret;
2301 }
2302
2303 static
2304 long lttng_event_notifier_group_ioctl(struct file *file, unsigned int cmd,
2305                 unsigned long arg)
2306 {
2307         switch (cmd) {
2308         case LTTNG_KERNEL_ABI_EVENT_NOTIFIER_GROUP_NOTIFICATION_FD:
2309         {
2310                 return lttng_abi_open_event_notifier_group_stream(file);
2311         }
2312         case LTTNG_KERNEL_ABI_EVENT_NOTIFIER_CREATE:
2313         {
2314                 struct lttng_kernel_abi_event_notifier uevent_notifier_param;
2315
2316                 if (copy_from_user(&uevent_notifier_param,
2317                                 (struct lttng_kernel_abi_event_notifier __user *) arg,
2318                                 sizeof(uevent_notifier_param)))
2319                         return -EFAULT;
2320                 return lttng_abi_create_event_notifier(file, &uevent_notifier_param);
2321         }
2322         case LTTNG_KERNEL_ABI_COUNTER:
2323         {
2324                 struct lttng_kernel_abi_counter_conf uerror_counter_conf;
2325
2326                 if (copy_from_user(&uerror_counter_conf,
2327                                 (struct lttng_kernel_abi_counter_conf __user *) arg,
2328                                 sizeof(uerror_counter_conf)))
2329                         return -EFAULT;
2330                 return lttng_abi_event_notifier_group_create_error_counter(file,
2331                                 &uerror_counter_conf);
2332         }
2333         default:
2334                 return -ENOIOCTLCMD;
2335         }
2336         return 0;
2337 }
2338
2339 static
2340 int lttng_event_notifier_group_release(struct inode *inode, struct file *file)
2341 {
2342         struct lttng_event_notifier_group *event_notifier_group =
2343                         file->private_data;
2344
2345         if (event_notifier_group)
2346                 lttng_event_notifier_group_destroy(event_notifier_group);
2347         return 0;
2348 }
2349
2350 static const struct file_operations lttng_event_notifier_group_fops = {
2351         .owner = THIS_MODULE,
2352         .release = lttng_event_notifier_group_release,
2353         .unlocked_ioctl = lttng_event_notifier_group_ioctl,
2354 #ifdef CONFIG_COMPAT
2355         .compat_ioctl = lttng_event_notifier_group_ioctl,
2356 #endif
2357 };
2358
2359 /**
2360  *      lttng_channel_ioctl - lttng syscall through ioctl
2361  *
2362  *      @file: the file
2363  *      @cmd: the command
2364  *      @arg: command arg
2365  *
2366  *      This ioctl implements lttng commands:
2367  *      LTTNG_KERNEL_ABI_STREAM
2368  *              Returns an event stream file descriptor or failure.
2369  *              (typically, one event stream records events from one CPU)
2370  *      LTTNG_KERNEL_ABI_EVENT
2371  *              Returns an event file descriptor or failure.
2372  *      LTTNG_KERNEL_ABI_CONTEXT
2373  *              Prepend a context field to each event in the channel
2374  *      LTTNG_KERNEL_ABI_ENABLE
2375  *              Enable recording for events in this channel (weak enable)
2376  *      LTTNG_KERNEL_ABI_DISABLE
2377  *              Disable recording for events in this channel (strong disable)
2378  *
2379  * Channel and event file descriptors also hold a reference on the session.
2380  */
2381 static
2382 long lttng_channel_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
2383 {
2384         struct lttng_kernel_channel_buffer *channel = file->private_data;
2385
2386         switch (cmd) {
2387         case LTTNG_KERNEL_ABI_OLD_STREAM:
2388         case LTTNG_KERNEL_ABI_STREAM:
2389                 return lttng_abi_open_stream(file);
2390         case LTTNG_KERNEL_ABI_OLD_EVENT:
2391         {
2392                 struct lttng_kernel_abi_event *uevent_param;
2393                 struct lttng_kernel_abi_old_event *old_uevent_param;
2394                 int ret;
2395
2396                 uevent_param = kmalloc(sizeof(struct lttng_kernel_abi_event),
2397                                 GFP_KERNEL);
2398                 if (!uevent_param) {
2399                         ret = -ENOMEM;
2400                         goto old_event_end;
2401                 }
2402                 old_uevent_param = kmalloc(
2403                                 sizeof(struct lttng_kernel_abi_old_event),
2404                                 GFP_KERNEL);
2405                 if (!old_uevent_param) {
2406                         ret = -ENOMEM;
2407                         goto old_event_error_free_param;
2408                 }
2409                 if (copy_from_user(old_uevent_param,
2410                                 (struct lttng_kernel_abi_old_event __user *) arg,
2411                                 sizeof(struct lttng_kernel_abi_old_event))) {
2412                         ret = -EFAULT;
2413                         goto old_event_error_free_old_param;
2414                 }
2415
2416                 memcpy(uevent_param->name, old_uevent_param->name,
2417                                 sizeof(uevent_param->name));
2418                 uevent_param->instrumentation =
2419                         old_uevent_param->instrumentation;
2420
2421                 switch (old_uevent_param->instrumentation) {
2422                 case LTTNG_KERNEL_ABI_KPROBE:
2423                         uevent_param->u.kprobe.addr =
2424                                 old_uevent_param->u.kprobe.addr;
2425                         uevent_param->u.kprobe.offset =
2426                                 old_uevent_param->u.kprobe.offset;
2427                         memcpy(uevent_param->u.kprobe.symbol_name,
2428                                 old_uevent_param->u.kprobe.symbol_name,
2429                                 sizeof(uevent_param->u.kprobe.symbol_name));
2430                         break;
2431                 case LTTNG_KERNEL_ABI_KRETPROBE:
2432                         uevent_param->u.kretprobe.addr =
2433                                 old_uevent_param->u.kretprobe.addr;
2434                         uevent_param->u.kretprobe.offset =
2435                                 old_uevent_param->u.kretprobe.offset;
2436                         memcpy(uevent_param->u.kretprobe.symbol_name,
2437                                 old_uevent_param->u.kretprobe.symbol_name,
2438                                 sizeof(uevent_param->u.kretprobe.symbol_name));
2439                         break;
2440                 case LTTNG_KERNEL_ABI_FUNCTION:
2441                         WARN_ON_ONCE(1);
2442                         /* Not implemented. */
2443                         break;
2444                 default:
2445                         break;
2446                 }
2447                 ret = lttng_abi_create_event(file, uevent_param);
2448
2449 old_event_error_free_old_param:
2450                 kfree(old_uevent_param);
2451 old_event_error_free_param:
2452                 kfree(uevent_param);
2453 old_event_end:
2454                 return ret;
2455         }
2456         case LTTNG_KERNEL_ABI_EVENT:
2457         {
2458                 struct lttng_kernel_abi_event uevent_param;
2459
2460                 if (copy_from_user(&uevent_param,
2461                                 (struct lttng_kernel_abi_event __user *) arg,
2462                                 sizeof(uevent_param)))
2463                         return -EFAULT;
2464                 return lttng_abi_create_event(file, &uevent_param);
2465         }
2466         case LTTNG_KERNEL_ABI_OLD_CONTEXT:
2467         {
2468                 struct lttng_kernel_abi_context *ucontext_param;
2469                 struct lttng_kernel_abi_old_context *old_ucontext_param;
2470                 int ret;
2471
2472                 ucontext_param = kmalloc(sizeof(struct lttng_kernel_abi_context),
2473                                 GFP_KERNEL);
2474                 if (!ucontext_param) {
2475                         ret = -ENOMEM;
2476                         goto old_ctx_end;
2477                 }
2478                 old_ucontext_param = kmalloc(sizeof(struct lttng_kernel_abi_old_context),
2479                                 GFP_KERNEL);
2480                 if (!old_ucontext_param) {
2481                         ret = -ENOMEM;
2482                         goto old_ctx_error_free_param;
2483                 }
2484
2485                 if (copy_from_user(old_ucontext_param,
2486                                 (struct lttng_kernel_abi_old_context __user *) arg,
2487                                 sizeof(struct lttng_kernel_abi_old_context))) {
2488                         ret = -EFAULT;
2489                         goto old_ctx_error_free_old_param;
2490                 }
2491                 ucontext_param->ctx = old_ucontext_param->ctx;
2492                 memcpy(ucontext_param->padding, old_ucontext_param->padding,
2493                                 sizeof(ucontext_param->padding));
2494                 /* only type that uses the union */
2495                 if (old_ucontext_param->ctx == LTTNG_KERNEL_ABI_CONTEXT_PERF_COUNTER) {
2496                         ucontext_param->u.perf_counter.type =
2497                                 old_ucontext_param->u.perf_counter.type;
2498                         ucontext_param->u.perf_counter.config =
2499                                 old_ucontext_param->u.perf_counter.config;
2500                         memcpy(ucontext_param->u.perf_counter.name,
2501                                 old_ucontext_param->u.perf_counter.name,
2502                                 sizeof(ucontext_param->u.perf_counter.name));
2503                 }
2504
2505                 ret = lttng_abi_add_context(file,
2506                                 ucontext_param,
2507                                 &channel->priv->ctx, channel->parent.session);
2508
2509 old_ctx_error_free_old_param:
2510                 kfree(old_ucontext_param);
2511 old_ctx_error_free_param:
2512                 kfree(ucontext_param);
2513 old_ctx_end:
2514                 return ret;
2515         }
2516         case LTTNG_KERNEL_ABI_CONTEXT:
2517         {
2518                 struct lttng_kernel_abi_context ucontext_param;
2519
2520                 if (copy_from_user(&ucontext_param,
2521                                 (struct lttng_kernel_abi_context __user *) arg,
2522                                 sizeof(ucontext_param)))
2523                         return -EFAULT;
2524                 return lttng_abi_add_context(file,
2525                                 &ucontext_param,
2526                                 &channel->priv->ctx, channel->parent.session);
2527         }
2528         case LTTNG_KERNEL_ABI_OLD_ENABLE:
2529         case LTTNG_KERNEL_ABI_ENABLE:
2530                 return lttng_channel_enable(channel);
2531         case LTTNG_KERNEL_ABI_OLD_DISABLE:
2532         case LTTNG_KERNEL_ABI_DISABLE:
2533                 return lttng_channel_disable(channel);
2534         case LTTNG_KERNEL_ABI_SYSCALL_MASK:
2535                 return lttng_channel_syscall_mask(channel,
2536                         (struct lttng_kernel_abi_syscall_mask __user *) arg);
2537         default:
2538                 return -ENOIOCTLCMD;
2539         }
2540 }
2541
2542 /**
2543  *      lttng_metadata_ioctl - lttng syscall through ioctl
2544  *
2545  *      @file: the file
2546  *      @cmd: the command
2547  *      @arg: command arg
2548  *
2549  *      This ioctl implements lttng commands:
2550  *      LTTNG_KERNEL_ABI_STREAM
2551  *              Returns an event stream file descriptor or failure.
2552  *
2553  * Channel and event file descriptors also hold a reference on the session.
2554  */
2555 static
2556 long lttng_metadata_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
2557 {
2558         switch (cmd) {
2559         case LTTNG_KERNEL_ABI_OLD_STREAM:
2560         case LTTNG_KERNEL_ABI_STREAM:
2561                 return lttng_abi_open_metadata_stream(file);
2562         default:
2563                 return -ENOIOCTLCMD;
2564         }
2565 }
2566
2567 /**
2568  *      lttng_channel_poll - lttng stream addition/removal monitoring
2569  *
2570  *      @file: the file
2571  *      @wait: poll table
2572  */
2573 unsigned int lttng_channel_poll(struct file *file, poll_table *wait)
2574 {
2575         struct lttng_kernel_channel_buffer *channel = file->private_data;
2576         unsigned int mask = 0;
2577
2578         if (file->f_mode & FMODE_READ) {
2579                 poll_wait_set_exclusive(wait);
2580                 poll_wait(file, channel->ops->priv->get_hp_wait_queue(channel->priv->rb_chan),
2581                           wait);
2582
2583                 if (channel->ops->priv->is_disabled(channel->priv->rb_chan))
2584                         return POLLERR;
2585                 if (channel->ops->priv->is_finalized(channel->priv->rb_chan))
2586                         return POLLHUP;
2587                 if (channel->ops->priv->buffer_has_read_closed_stream(channel->priv->rb_chan))
2588                         return POLLIN | POLLRDNORM;
2589                 return 0;
2590         }
2591         return mask;
2592
2593 }
2594
2595 static
2596 int lttng_channel_release(struct inode *inode, struct file *file)
2597 {
2598         struct lttng_kernel_channel_buffer *channel = file->private_data;
2599
2600         if (channel)
2601                 fput(channel->parent.session->priv->file);
2602         return 0;
2603 }
2604
2605 static
2606 int lttng_metadata_channel_release(struct inode *inode, struct file *file)
2607 {
2608         struct lttng_kernel_channel_buffer *channel = file->private_data;
2609
2610         if (channel) {
2611                 fput(channel->parent.session->priv->file);
2612                 lttng_metadata_channel_destroy(channel);
2613         }
2614
2615         return 0;
2616 }
2617
2618 static const struct file_operations lttng_channel_fops = {
2619         .owner = THIS_MODULE,
2620         .release = lttng_channel_release,
2621         .poll = lttng_channel_poll,
2622         .unlocked_ioctl = lttng_channel_ioctl,
2623 #ifdef CONFIG_COMPAT
2624         .compat_ioctl = lttng_channel_ioctl,
2625 #endif
2626 };
2627
2628 static const struct file_operations lttng_metadata_fops = {
2629         .owner = THIS_MODULE,
2630         .release = lttng_metadata_channel_release,
2631         .unlocked_ioctl = lttng_metadata_ioctl,
2632 #ifdef CONFIG_COMPAT
2633         .compat_ioctl = lttng_metadata_ioctl,
2634 #endif
2635 };
2636
2637 /**
2638  *      lttng_event_recorder_event_ioctl - lttng syscall through ioctl
2639  *
2640  *      @file: the file
2641  *      @cmd: the command
2642  *      @arg: command arg
2643  *
2644  *      This ioctl implements lttng commands:
2645  *      LTTNG_KERNEL_ABI_CONTEXT
2646  *              Prepend a context field to each record of this event
2647  *      LTTNG_KERNEL_ABI_ENABLE
2648  *              Enable recording for this event (weak enable)
2649  *      LTTNG_KERNEL_ABI_DISABLE
2650  *              Disable recording for this event (strong disable)
2651  */
2652 static
2653 long lttng_event_recorder_event_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
2654 {
2655         struct lttng_kernel_event_recorder *event_recorder = file->private_data;
2656
2657         switch (cmd) {
2658         case LTTNG_KERNEL_ABI_OLD_CONTEXT:
2659         {
2660                 /* Not implemented */
2661                 return -ENOSYS;
2662         }
2663         case LTTNG_KERNEL_ABI_CONTEXT:
2664         {
2665                 /* Not implemented */
2666                 return -ENOSYS;
2667         }
2668         case LTTNG_KERNEL_ABI_OLD_ENABLE:
2669         case LTTNG_KERNEL_ABI_ENABLE:
2670                 return lttng_event_enable(&event_recorder->parent);
2671         case LTTNG_KERNEL_ABI_OLD_DISABLE:
2672         case LTTNG_KERNEL_ABI_DISABLE:
2673                 return lttng_event_disable(&event_recorder->parent);
2674         case LTTNG_KERNEL_ABI_FILTER:
2675                 return -EINVAL;
2676         case LTTNG_KERNEL_ABI_ADD_CALLSITE:
2677                 return lttng_event_add_callsite(&event_recorder->parent,
2678                         (struct lttng_kernel_abi_event_callsite __user *) arg);
2679         default:
2680                 return -ENOIOCTLCMD;
2681         }
2682 }
2683
2684 /**
2685  *      lttng_event_recorder_enabler_ioctl - lttng syscall through ioctl
2686  *
2687  *      @file: the file
2688  *      @cmd: the command
2689  *      @arg: command arg
2690  *
2691  *      This ioctl implements lttng commands:
2692  *      LTTNG_KERNEL_ABI_CONTEXT
2693  *              Prepend a context field to each record of this event
2694  *      LTTNG_KERNEL_ABI_ENABLE
2695  *              Enable recording for this event (weak enable)
2696  *      LTTNG_KERNEL_ABI_DISABLE
2697  *              Disable recording for this event (strong disable)
2698  */
2699 static
2700 long lttng_event_recorder_enabler_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
2701 {
2702         struct lttng_event_recorder_enabler *event_enabler = file->private_data;
2703
2704         switch (cmd) {
2705         case LTTNG_KERNEL_ABI_OLD_CONTEXT:
2706         {
2707                 /* Not implemented */
2708                 return -ENOSYS;
2709         }
2710         case LTTNG_KERNEL_ABI_CONTEXT:
2711         {
2712                 /* Not implemented */
2713                 return -ENOSYS;
2714         }
2715         case LTTNG_KERNEL_ABI_OLD_ENABLE:
2716         case LTTNG_KERNEL_ABI_ENABLE:
2717                 return lttng_event_enabler_enable(&event_enabler->parent);
2718         case LTTNG_KERNEL_ABI_OLD_DISABLE:
2719         case LTTNG_KERNEL_ABI_DISABLE:
2720                 return lttng_event_enabler_disable(&event_enabler->parent);
2721         case LTTNG_KERNEL_ABI_FILTER:
2722                 return lttng_event_enabler_attach_filter_bytecode(&event_enabler->parent,
2723                         (struct lttng_kernel_abi_filter_bytecode __user *) arg);
2724         case LTTNG_KERNEL_ABI_ADD_CALLSITE:
2725                 return -EINVAL;
2726         default:
2727                 return -ENOIOCTLCMD;
2728         }
2729 }
2730
2731 static
2732 int lttng_event_recorder_event_release(struct inode *inode, struct file *file)
2733 {
2734         struct lttng_kernel_event_recorder *event = file->private_data;
2735
2736         if (event)
2737                 fput(event->chan->priv->parent.file);
2738         return 0;
2739 }
2740
2741 static
2742 int lttng_event_recorder_enabler_release(struct inode *inode, struct file *file)
2743 {
2744         struct lttng_event_recorder_enabler *event_enabler = file->private_data;
2745
2746         if (event_enabler)
2747                 fput(event_enabler->chan->priv->parent.file);
2748         return 0;
2749 }
2750
2751 static const struct file_operations lttng_event_recorder_event_fops = {
2752         .owner = THIS_MODULE,
2753         .release = lttng_event_recorder_event_release,
2754         .unlocked_ioctl = lttng_event_recorder_event_ioctl,
2755 #ifdef CONFIG_COMPAT
2756         .compat_ioctl = lttng_event_recorder_event_ioctl,
2757 #endif
2758 };
2759
2760 static const struct file_operations lttng_event_recorder_enabler_fops = {
2761         .owner = THIS_MODULE,
2762         .release = lttng_event_recorder_enabler_release,
2763         .unlocked_ioctl = lttng_event_recorder_enabler_ioctl,
2764 #ifdef CONFIG_COMPAT
2765         .compat_ioctl = lttng_event_recorder_enabler_ioctl,
2766 #endif
2767 };
2768
2769 static int put_u64(uint64_t val, unsigned long arg)
2770 {
2771         return put_user(val, (uint64_t __user *) arg);
2772 }
2773
2774 static int put_u32(uint32_t val, unsigned long arg)
2775 {
2776         return put_user(val, (uint32_t __user *) arg);
2777 }
2778
2779 static long lttng_stream_ring_buffer_ioctl(struct file *filp,
2780                 unsigned int cmd, unsigned long arg)
2781 {
2782         struct lttng_kernel_ring_buffer *buf = filp->private_data;
2783         struct lttng_kernel_ring_buffer_channel *chan = buf->backend.chan;
2784         const struct lttng_kernel_ring_buffer_config *config = &chan->backend.config;
2785         const struct lttng_kernel_channel_buffer_ops *ops = chan->backend.priv_ops;
2786         int ret;
2787
2788         if (atomic_read(&chan->record_disabled))
2789                 return -EIO;
2790
2791         switch (cmd) {
2792         case LTTNG_KERNEL_ABI_RING_BUFFER_GET_TIMESTAMP_BEGIN:
2793         {
2794                 uint64_t ts;
2795
2796                 ret = ops->priv->timestamp_begin(config, buf, &ts);
2797                 if (ret < 0)
2798                         goto error;
2799                 return put_u64(ts, arg);
2800         }
2801         case LTTNG_KERNEL_ABI_RING_BUFFER_GET_TIMESTAMP_END:
2802         {
2803                 uint64_t ts;
2804
2805                 ret = ops->priv->timestamp_end(config, buf, &ts);
2806                 if (ret < 0)
2807                         goto error;
2808                 return put_u64(ts, arg);
2809         }
2810         case LTTNG_KERNEL_ABI_RING_BUFFER_GET_EVENTS_DISCARDED:
2811         {
2812                 uint64_t ed;
2813
2814                 ret = ops->priv->events_discarded(config, buf, &ed);
2815                 if (ret < 0)
2816                         goto error;
2817                 return put_u64(ed, arg);
2818         }
2819         case LTTNG_KERNEL_ABI_RING_BUFFER_GET_CONTENT_SIZE:
2820         {
2821                 uint64_t cs;
2822
2823                 ret = ops->priv->content_size(config, buf, &cs);
2824                 if (ret < 0)
2825                         goto error;
2826                 return put_u64(cs, arg);
2827         }
2828         case LTTNG_KERNEL_ABI_RING_BUFFER_GET_PACKET_SIZE:
2829         {
2830                 uint64_t ps;
2831
2832                 ret = ops->priv->packet_size(config, buf, &ps);
2833                 if (ret < 0)
2834                         goto error;
2835                 return put_u64(ps, arg);
2836         }
2837         case LTTNG_KERNEL_ABI_RING_BUFFER_GET_STREAM_ID:
2838         {
2839                 uint64_t si;
2840
2841                 ret = ops->priv->stream_id(config, buf, &si);
2842                 if (ret < 0)
2843                         goto error;
2844                 return put_u64(si, arg);
2845         }
2846         case LTTNG_KERNEL_ABI_RING_BUFFER_GET_CURRENT_TIMESTAMP:
2847         {
2848                 uint64_t ts;
2849
2850                 ret = ops->priv->current_timestamp(config, buf, &ts);
2851                 if (ret < 0)
2852                         goto error;
2853                 return put_u64(ts, arg);
2854         }
2855         case LTTNG_KERNEL_ABI_RING_BUFFER_GET_SEQ_NUM:
2856         {
2857                 uint64_t seq;
2858
2859                 ret = ops->priv->sequence_number(config, buf, &seq);
2860                 if (ret < 0)
2861                         goto error;
2862                 return put_u64(seq, arg);
2863         }
2864         case LTTNG_KERNEL_ABI_RING_BUFFER_INSTANCE_ID:
2865         {
2866                 uint64_t id;
2867
2868                 ret = ops->priv->instance_id(config, buf, &id);
2869                 if (ret < 0)
2870                         goto error;
2871                 return put_u64(id, arg);
2872         }
2873         default:
2874                 return lib_ring_buffer_file_operations.unlocked_ioctl(filp,
2875                                 cmd, arg);
2876         }
2877
2878 error:
2879         return -ENOSYS;
2880 }
2881
2882 #ifdef CONFIG_COMPAT
2883 static long lttng_stream_ring_buffer_compat_ioctl(struct file *filp,
2884                 unsigned int cmd, unsigned long arg)
2885 {
2886         struct lttng_kernel_ring_buffer *buf = filp->private_data;
2887         struct lttng_kernel_ring_buffer_channel *chan = buf->backend.chan;
2888         const struct lttng_kernel_ring_buffer_config *config = &chan->backend.config;
2889         const struct lttng_kernel_channel_buffer_ops *ops = chan->backend.priv_ops;
2890         int ret;
2891
2892         if (atomic_read(&chan->record_disabled))
2893                 return -EIO;
2894
2895         switch (cmd) {
2896         case LTTNG_KERNEL_ABI_RING_BUFFER_COMPAT_GET_TIMESTAMP_BEGIN:
2897         {
2898                 uint64_t ts;
2899
2900                 ret = ops->priv->timestamp_begin(config, buf, &ts);
2901                 if (ret < 0)
2902                         goto error;
2903                 return put_u64(ts, arg);
2904         }
2905         case LTTNG_KERNEL_ABI_RING_BUFFER_COMPAT_GET_TIMESTAMP_END:
2906         {
2907                 uint64_t ts;
2908
2909                 ret = ops->priv->timestamp_end(config, buf, &ts);
2910                 if (ret < 0)
2911                         goto error;
2912                 return put_u64(ts, arg);
2913         }
2914         case LTTNG_KERNEL_ABI_RING_BUFFER_COMPAT_GET_EVENTS_DISCARDED:
2915         {
2916                 uint64_t ed;
2917
2918                 ret = ops->priv->events_discarded(config, buf, &ed);
2919                 if (ret < 0)
2920                         goto error;
2921                 return put_u64(ed, arg);
2922         }
2923         case LTTNG_KERNEL_ABI_RING_BUFFER_COMPAT_GET_CONTENT_SIZE:
2924         {
2925                 uint64_t cs;
2926
2927                 ret = ops->priv->content_size(config, buf, &cs);
2928                 if (ret < 0)
2929                         goto error;
2930                 return put_u64(cs, arg);
2931         }
2932         case LTTNG_KERNEL_ABI_RING_BUFFER_COMPAT_GET_PACKET_SIZE:
2933         {
2934                 uint64_t ps;
2935
2936                 ret = ops->priv->packet_size(config, buf, &ps);
2937                 if (ret < 0)
2938                         goto error;
2939                 return put_u64(ps, arg);
2940         }
2941         case LTTNG_KERNEL_ABI_RING_BUFFER_COMPAT_GET_STREAM_ID:
2942         {
2943                 uint64_t si;
2944
2945                 ret = ops->priv->stream_id(config, buf, &si);
2946                 if (ret < 0)
2947                         goto error;
2948                 return put_u64(si, arg);
2949         }
2950         case LTTNG_KERNEL_ABI_RING_BUFFER_GET_CURRENT_TIMESTAMP:
2951         {
2952                 uint64_t ts;
2953
2954                 ret = ops->priv->current_timestamp(config, buf, &ts);
2955                 if (ret < 0)
2956                         goto error;
2957                 return put_u64(ts, arg);
2958         }
2959         case LTTNG_KERNEL_ABI_RING_BUFFER_COMPAT_GET_SEQ_NUM:
2960         {
2961                 uint64_t seq;
2962
2963                 ret = ops->priv->sequence_number(config, buf, &seq);
2964                 if (ret < 0)
2965                         goto error;
2966                 return put_u64(seq, arg);
2967         }
2968         case LTTNG_KERNEL_ABI_RING_BUFFER_COMPAT_INSTANCE_ID:
2969         {
2970                 uint64_t id;
2971
2972                 ret = ops->priv->instance_id(config, buf, &id);
2973                 if (ret < 0)
2974                         goto error;
2975                 return put_u64(id, arg);
2976         }
2977         default:
2978                 return lib_ring_buffer_file_operations.compat_ioctl(filp,
2979                                 cmd, arg);
2980         }
2981
2982 error:
2983         return -ENOSYS;
2984 }
2985 #endif /* CONFIG_COMPAT */
2986
2987 static void lttng_stream_override_ring_buffer_fops(void)
2988 {
2989         lttng_stream_ring_buffer_file_operations.owner = THIS_MODULE;
2990         lttng_stream_ring_buffer_file_operations.open =
2991                 lib_ring_buffer_file_operations.open;
2992         lttng_stream_ring_buffer_file_operations.release =
2993                 lib_ring_buffer_file_operations.release;
2994         lttng_stream_ring_buffer_file_operations.poll =
2995                 lib_ring_buffer_file_operations.poll;
2996         lttng_stream_ring_buffer_file_operations.splice_read =
2997                 lib_ring_buffer_file_operations.splice_read;
2998         lttng_stream_ring_buffer_file_operations.mmap =
2999                 lib_ring_buffer_file_operations.mmap;
3000         lttng_stream_ring_buffer_file_operations.unlocked_ioctl =
3001                 lttng_stream_ring_buffer_ioctl;
3002         lttng_stream_ring_buffer_file_operations.llseek =
3003                 lib_ring_buffer_file_operations.llseek;
3004 #ifdef CONFIG_COMPAT
3005         lttng_stream_ring_buffer_file_operations.compat_ioctl =
3006                 lttng_stream_ring_buffer_compat_ioctl;
3007 #endif
3008 }
3009
3010 int __init lttng_abi_init(void)
3011 {
3012         int ret = 0;
3013
3014         wrapper_vmalloc_sync_mappings();
3015         lttng_clock_ref();
3016
3017         ret = lttng_tp_mempool_init();
3018         if (ret) {
3019                 goto error;
3020         }
3021
3022         lttng_proc_dentry = proc_create_data("lttng", S_IRUSR | S_IWUSR, NULL,
3023                                         &lttng_proc_ops, NULL);
3024
3025         if (!lttng_proc_dentry) {
3026                 printk(KERN_ERR "LTTng: Error creating control file\n");
3027                 ret = -ENOMEM;
3028                 goto error;
3029         }
3030         lttng_stream_override_ring_buffer_fops();
3031         return 0;
3032
3033 error:
3034         lttng_tp_mempool_destroy();
3035         lttng_clock_unref();
3036         return ret;
3037 }
3038
3039 /* No __exit annotation because used by init error path too. */
3040 void lttng_abi_exit(void)
3041 {
3042         lttng_tp_mempool_destroy();
3043         lttng_clock_unref();
3044         if (lttng_proc_dentry)
3045                 remove_proc_entry("lttng", NULL);
3046 }