src/bin/lttng-sessiond/kernel.cpp

   1 /*
   2  * Copyright (C) 2011 EfficiOS Inc.
   3  *
   4  * SPDX-License-Identifier: GPL-2.0-only
   5  *
   6  */
   7
   8 #define _LGPL_SOURCE
   9 #include "condition-internal.hpp"
  10 #include "consumer.hpp"
  11 #include "event-notifier-error-accounting.hpp"
  12 #include "kern-modules.hpp"
  13 #include "kernel-consumer.hpp"
  14 #include "kernel.hpp"
  15 #include "lttng-sessiond.hpp"
  16 #include "lttng-syscall.hpp"
  17 #include "modprobe.hpp"
  18 #include "notification-thread-commands.hpp"
  19 #include "sessiond-config.hpp"
  20 #include "tracker.hpp"
  21 #include "utils.hpp"
  22
  23 #include <common/common.hpp>
  24 #include <common/hashtable/utils.hpp>
  25 #include <common/kernel-ctl/kernel-ctl.hpp>
  26 #include <common/kernel-ctl/kernel-ioctl.hpp>
  27 #include <common/sessiond-comm/sessiond-comm.hpp>
  28 #include <common/trace-chunk.hpp>
  29 #include <common/tracker.hpp>
  30 #include <common/urcu.hpp>
  31 #include <common/utils.hpp>
  32
  33 #include <lttng/condition/event-rule-matches-internal.hpp>
  34 #include <lttng/condition/event-rule-matches.h>
  35 #include <lttng/event-rule/event-rule-internal.hpp>
  36 #include <lttng/event-rule/event-rule.h>
  37 #include <lttng/event-rule/kernel-uprobe-internal.hpp>
  38 #include <lttng/event.h>
  39 #include <lttng/lttng-error.h>
  40 #include <lttng/tracker.h>
  41 #include <lttng/userspace-probe-internal.hpp>
  42 #include <lttng/userspace-probe.h>
  43
  44 #include <fcntl.h>
  45 #include <inttypes.h>
  46 #include <stdio.h>
  47 #include <stdlib.h>
  48 #include <string.h>
  49 #include <sys/types.h>
  50 #include <unistd.h>
  51
  52 namespace {
  53 /*
  54  * On some architectures, calling convention details are embedded in the symbol
  55  * addresses. Uprobe requires a "clean" symbol offset (or at least, an address
  56  * where an instruction boundary would be legal) to add
  57  * instrumentation. sanitize_uprobe_offset implements that sanitization logic on
  58  * a per-architecture basis.
  59  */
  60 #if defined(__arm__) || defined(__aarch64__)
  61 static inline uint64_t sanitize_uprobe_offset(uint64_t raw_offset)
  62 {
  63         /*
  64          * The least significant bit is used when branching to switch to thumb
  65          * ISA. However, it's an invalid address for us; mask the least
  66          * significant bit.
  67          */
  68         return raw_offset &= ~0b1;
  69 }
  70 #else /* defined(__arm__) || defined(__aarch64__) */
  71 static inline uint64_t sanitize_uprobe_offset(uint64_t raw_offset)
  72 {
  73         return raw_offset;
  74 }
  75 #endif
  76
  77 /*
  78  * Key used to reference a channel between the sessiond and the consumer. This
  79  * is only read and updated with the session_list lock held.
  80  */
  81 uint64_t next_kernel_channel_key;
  82
  83 const char *module_proc_lttng = "/proc/lttng";
  84
  85 int kernel_tracer_fd = -1;
  86 nonstd::optional<enum lttng_kernel_tracer_status> kernel_tracer_status = nonstd::nullopt;
  87 int kernel_tracer_event_notifier_group_fd = -1;
  88 int kernel_tracer_event_notifier_group_notification_fd = -1;
  89 struct cds_lfht *kernel_token_to_event_notifier_rule_ht;
  90 } /* namespace */
  91
  92 /*
  93  * Add context on a kernel channel.
  94  *
  95  * Assumes the ownership of ctx.
  96  */
  97 int kernel_add_channel_context(struct ltt_kernel_channel *chan, struct ltt_kernel_context *ctx)
  98 {
  99         int ret;
 100
 101         LTTNG_ASSERT(chan);
 102         LTTNG_ASSERT(ctx);
 103
 104         DBG("Adding context to channel %s", chan->channel->name);
 105         ret = kernctl_add_context(chan->fd, &ctx->ctx);
 106         if (ret < 0) {
 107                 switch (-ret) {
 108                 case ENOSYS:
 109                         /* Exists but not available for this kernel */
 110                         ret = LTTNG_ERR_KERN_CONTEXT_UNAVAILABLE;
 111                         goto error;
 112                 case EEXIST:
 113                         /* If EEXIST, we just ignore the error */
 114                         ret = 0;
 115                         goto end;
 116                 default:
 117                         PERROR("add context ioctl");
 118                         ret = LTTNG_ERR_KERN_CONTEXT_FAIL;
 119                         goto error;
 120                 }
 121         }
 122         ret = 0;
 123
 124 end:
 125         cds_list_add_tail(&ctx->list, &chan->ctx_list);
 126         ctx->in_list = true;
 127         ctx = nullptr;
 128 error:
 129         if (ctx) {
 130                 trace_kernel_destroy_context(ctx);
 131         }
 132         return ret;
 133 }
 134
 135 /*
 136  * Create a new kernel session, register it to the kernel tracer and add it to
 137  * the session daemon session.
 138  */
 139 int kernel_create_session(struct ltt_session *session)
 140 {
 141         int ret;
 142         struct ltt_kernel_session *lks;
 143
 144         LTTNG_ASSERT(session);
 145
 146         /* Allocate data structure */
 147         lks = trace_kernel_create_session();
 148         if (lks == nullptr) {
 149                 ret = -1;
 150                 goto error;
 151         }
 152
 153         /* Kernel tracer session creation */
 154         ret = kernctl_create_session(kernel_tracer_fd);
 155         if (ret < 0) {
 156                 PERROR("ioctl kernel create session");
 157                 goto error;
 158         }
 159
 160         lks->fd = ret;
 161         /* Prevent fd duplication after execlp() */
 162         ret = fcntl(lks->fd, F_SETFD, FD_CLOEXEC);
 163         if (ret < 0) {
 164                 PERROR("fcntl session fd");
 165         }
 166
 167         lks->id = session->id;
 168         lks->consumer_fds_sent = 0;
 169         session->kernel_session = lks;
 170
 171         DBG("Kernel session created (fd: %d)", lks->fd);
 172
 173         /*
 174          * This is necessary since the creation time is present in the session
 175          * name when it is generated.
 176          */
 177         if (session->has_auto_generated_name) {
 178                 ret = kernctl_session_set_name(lks->fd, DEFAULT_SESSION_NAME);
 179         } else {
 180                 ret = kernctl_session_set_name(lks->fd, session->name);
 181         }
 182         if (ret) {
 183                 WARN("Could not set kernel session name for session %" PRIu64 " name: %s",
 184                      session->id,
 185                      session->name);
 186         }
 187
 188         ret = kernctl_session_set_creation_time(lks->fd, session->creation_time);
 189         if (ret) {
 190                 WARN("Could not set kernel session creation time for session %" PRIu64 " name: %s",
 191                      session->id,
 192                      session->name);
 193         }
 194
 195         return 0;
 196
 197 error:
 198         if (lks) {
 199                 trace_kernel_destroy_session(lks);
 200                 trace_kernel_free_session(lks);
 201         }
 202         return ret;
 203 }
 204
 205 /*
 206  * Create a kernel channel, register it to the kernel tracer and add it to the
 207  * kernel session.
 208  */
 209 int kernel_create_channel(struct ltt_kernel_session *session, struct lttng_channel *chan)
 210 {
 211         int ret;
 212         struct ltt_kernel_channel *lkc;
 213
 214         LTTNG_ASSERT(session);
 215         LTTNG_ASSERT(chan);
 216
 217         /* Allocate kernel channel */
 218         lkc = trace_kernel_create_channel(chan);
 219         if (lkc == nullptr) {
 220                 goto error;
 221         }
 222
 223         DBG3("Kernel create channel %s with attr: %d, %" PRIu64 ", %" PRIu64 ", %u, %u, %d, %d",
 224              chan->name,
 225              lkc->channel->attr.overwrite,
 226              lkc->channel->attr.subbuf_size,
 227              lkc->channel->attr.num_subbuf,
 228              lkc->channel->attr.switch_timer_interval,
 229              lkc->channel->attr.read_timer_interval,
 230              lkc->channel->attr.live_timer_interval,
 231              lkc->channel->attr.output);
 232
 233         /* Kernel tracer channel creation */
 234         ret = kernctl_create_channel(session->fd, &lkc->channel->attr);
 235         if (ret < 0) {
 236                 PERROR("ioctl kernel create channel");
 237                 goto error;
 238         }
 239
 240         /* Setup the channel fd */
 241         lkc->fd = ret;
 242         /* Prevent fd duplication after execlp() */
 243         ret = fcntl(lkc->fd, F_SETFD, FD_CLOEXEC);
 244         if (ret < 0) {
 245                 PERROR("fcntl session fd");
 246         }
 247
 248         /* Add channel to session */
 249         cds_list_add(&lkc->list, &session->channel_list.head);
 250         session->channel_count++;
 251         lkc->session = session;
 252         lkc->key = ++next_kernel_channel_key;
 253
 254         DBG("Kernel channel %s created (fd: %d, key: %" PRIu64 ")",
 255             lkc->channel->name,
 256             lkc->fd,
 257             lkc->key);
 258
 259         return 0;
 260
 261 error:
 262         if (lkc) {
 263                 free(lkc->channel);
 264                 free(lkc);
 265         }
 266         return -1;
 267 }
 268
 269 /*
 270  * Create a kernel event notifier group, register it to the kernel tracer and
 271  * add it to the kernel session.
 272  */
 273 static int kernel_create_event_notifier_group(int *event_notifier_group_fd)
 274 {
 275         int ret;
 276         int local_fd = -1;
 277
 278         LTTNG_ASSERT(event_notifier_group_fd);
 279
 280         /* Kernel event notifier group creation. */
 281         ret = kernctl_create_event_notifier_group(kernel_tracer_fd);
 282         if (ret < 0) {
 283                 PERROR("Failed to create kernel event notifier group");
 284                 ret = -1;
 285                 goto error;
 286         }
 287
 288         local_fd = ret;
 289
 290         /* Prevent fd duplication after execlp(). */
 291         ret = fcntl(local_fd, F_SETFD, FD_CLOEXEC);
 292         if (ret < 0) {
 293                 PERROR("Failed to set FD_CLOEXEC on kernel event notifier group file descriptor: fd = %d",
 294                        local_fd);
 295                 goto error;
 296         }
 297
 298         DBG("Created kernel event notifier group: fd = %d", local_fd);
 299         *event_notifier_group_fd = local_fd;
 300         local_fd = -1;
 301         ret = 0;
 302 error:
 303         if (local_fd >= 0) {
 304                 ret = close(local_fd);
 305                 if (ret) {
 306                         PERROR("Failed to close kernel event notifier group file descriptor: fd = %d",
 307                                local_fd);
 308                 }
 309         }
 310
 311         return ret;
 312 }
 313
 314 /*
 315  * Compute the offset of the instrumentation byte in the binary based on the
 316  * function probe location using the ELF lookup method.
 317  *
 318  * Returns 0 on success and set the offset out parameter to the offset of the
 319  * elf symbol
 320  * Returns -1 on error
 321  */
 322 static int extract_userspace_probe_offset_function_elf(
 323         const struct lttng_userspace_probe_location *probe_location,
 324         uid_t uid,
 325         gid_t gid,
 326         uint64_t *offset)
 327 {
 328         int fd;
 329         int ret = 0;
 330         const char *symbol = nullptr;
 331         const struct lttng_userspace_probe_location_lookup_method *lookup = nullptr;
 332         enum lttng_userspace_probe_location_lookup_method_type lookup_method_type;
 333
 334         LTTNG_ASSERT(lttng_userspace_probe_location_get_type(probe_location) ==
 335                      LTTNG_USERSPACE_PROBE_LOCATION_TYPE_FUNCTION);
 336
 337         lookup = lttng_userspace_probe_location_get_lookup_method(probe_location);
 338         if (!lookup) {
 339                 ret = -1;
 340                 goto end;
 341         }
 342
 343         lookup_method_type = lttng_userspace_probe_location_lookup_method_get_type(lookup);
 344
 345         LTTNG_ASSERT(lookup_method_type ==
 346                      LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_FUNCTION_ELF);
 347
 348         symbol = lttng_userspace_probe_location_function_get_function_name(probe_location);
 349         if (!symbol) {
 350                 ret = -1;
 351                 goto end;
 352         }
 353
 354         fd = lttng_userspace_probe_location_function_get_binary_fd(probe_location);
 355         if (fd < 0) {
 356                 ret = -1;
 357                 goto end;
 358         }
 359
 360         ret = run_as_extract_elf_symbol_offset(fd, symbol, uid, gid, offset);
 361         if (ret < 0) {
 362                 DBG("userspace probe offset calculation failed for "
 363                     "function %s",
 364                     symbol);
 365                 goto end;
 366         }
 367
 368         DBG("userspace probe elf offset for %s is 0x%jd", symbol, (intmax_t) (*offset));
 369 end:
 370         return ret;
 371 }
 372
 373 /*
 374  * Compute the offsets of the instrumentation bytes in the binary based on the
 375  * tracepoint probe location using the SDT lookup method. This function
 376  * allocates the offsets buffer, the caller must free it.
 377  *
 378  * Returns 0 on success and set the offset out parameter to the offsets of the
 379  * SDT tracepoint.
 380  * Returns -1 on error.
 381  */
 382 static int extract_userspace_probe_offset_tracepoint_sdt(
 383         const struct lttng_userspace_probe_location *probe_location,
 384         uid_t uid,
 385         gid_t gid,
 386         uint64_t **offsets,
 387         uint32_t *offsets_count)
 388 {
 389         enum lttng_userspace_probe_location_lookup_method_type lookup_method_type;
 390         const struct lttng_userspace_probe_location_lookup_method *lookup = nullptr;
 391         const char *probe_name = nullptr, *provider_name = nullptr;
 392         int ret = 0;
 393         int fd, i;
 394
 395         LTTNG_ASSERT(lttng_userspace_probe_location_get_type(probe_location) ==
 396                      LTTNG_USERSPACE_PROBE_LOCATION_TYPE_TRACEPOINT);
 397
 398         lookup = lttng_userspace_probe_location_get_lookup_method(probe_location);
 399         if (!lookup) {
 400                 ret = -1;
 401                 goto end;
 402         }
 403
 404         lookup_method_type = lttng_userspace_probe_location_lookup_method_get_type(lookup);
 405
 406         LTTNG_ASSERT(lookup_method_type ==
 407                      LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_TRACEPOINT_SDT);
 408
 409         probe_name = lttng_userspace_probe_location_tracepoint_get_probe_name(probe_location);
 410         if (!probe_name) {
 411                 ret = -1;
 412                 goto end;
 413         }
 414
 415         provider_name = lttng_userspace_probe_location_tracepoint_get_provider_name(probe_location);
 416         if (!provider_name) {
 417                 ret = -1;
 418                 goto end;
 419         }
 420
 421         fd = lttng_userspace_probe_location_tracepoint_get_binary_fd(probe_location);
 422         if (fd < 0) {
 423                 ret = -1;
 424                 goto end;
 425         }
 426
 427         ret = run_as_extract_sdt_probe_offsets(
 428                 fd, provider_name, probe_name, uid, gid, offsets, offsets_count);
 429         if (ret < 0) {
 430                 DBG("userspace probe offset calculation failed for sdt "
 431                     "probe %s:%s",
 432                     provider_name,
 433                     probe_name);
 434                 goto end;
 435         }
 436
 437         if (*offsets_count == 0) {
 438                 DBG("no userspace probe offset found");
 439                 goto end;
 440         }
 441
 442         DBG("%u userspace probe SDT offsets found for %s:%s at:",
 443             *offsets_count,
 444             provider_name,
 445             probe_name);
 446         for (i = 0; i < *offsets_count; i++) {
 447                 DBG("\t0x%jd", (intmax_t) ((*offsets)[i]));
 448         }
 449 end:
 450         return ret;
 451 }
 452
 453 static int userspace_probe_add_callsite(const struct lttng_userspace_probe_location *location,
 454                                         uid_t uid,
 455                                         gid_t gid,
 456                                         int fd)
 457 {
 458         const struct lttng_userspace_probe_location_lookup_method *lookup_method = nullptr;
 459         enum lttng_userspace_probe_location_lookup_method_type type;
 460         int ret;
 461
 462         lookup_method = lttng_userspace_probe_location_get_lookup_method(location);
 463         if (!lookup_method) {
 464                 ret = -1;
 465                 goto end;
 466         }
 467
 468         type = lttng_userspace_probe_location_lookup_method_get_type(lookup_method);
 469         switch (type) {
 470         case LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_FUNCTION_ELF:
 471         {
 472                 struct lttng_kernel_abi_event_callsite callsite;
 473                 uint64_t offset;
 474
 475                 ret = extract_userspace_probe_offset_function_elf(location, uid, gid, &offset);
 476                 if (ret) {
 477                         ret = LTTNG_ERR_PROBE_LOCATION_INVAL;
 478                         goto end;
 479                 }
 480
 481                 callsite.u.uprobe.offset = sanitize_uprobe_offset(offset);
 482                 ret = kernctl_add_callsite(fd, &callsite);
 483                 if (ret) {
 484                         WARN("Failed to add callsite to ELF userspace probe.");
 485                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 486                         goto end;
 487                 }
 488                 break;
 489         }
 490         case LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_TRACEPOINT_SDT:
 491         {
 492                 int i;
 493                 uint64_t *offsets = nullptr;
 494                 uint32_t offsets_count;
 495                 struct lttng_kernel_abi_event_callsite callsite;
 496
 497                 /*
 498                  * This call allocates the offsets buffer. This buffer must be freed
 499                  * by the caller
 500                  */
 501                 ret = extract_userspace_probe_offset_tracepoint_sdt(
 502                         location, uid, gid, &offsets, &offsets_count);
 503                 if (ret) {
 504                         ret = LTTNG_ERR_PROBE_LOCATION_INVAL;
 505                         goto end;
 506                 }
 507                 for (i = 0; i < offsets_count; i++) {
 508                         callsite.u.uprobe.offset = sanitize_uprobe_offset(offsets[i]);
 509                         ret = kernctl_add_callsite(fd, &callsite);
 510                         if (ret) {
 511                                 WARN("Failed to add callsite to SDT userspace probe");
 512                                 ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 513                                 free(offsets);
 514                                 goto end;
 515                         }
 516                 }
 517                 free(offsets);
 518                 break;
 519         }
 520         default:
 521                 ret = LTTNG_ERR_PROBE_LOCATION_INVAL;
 522                 goto end;
 523         }
 524 end:
 525         return ret;
 526 }
 527
 528 /*
 529  * Extract the offsets of the instrumentation point for the different lookup
 530  * methods.
 531  */
 532 static int userspace_probe_event_add_callsites(struct lttng_event *ev,
 533                                                struct ltt_kernel_session *session,
 534                                                int fd)
 535 {
 536         int ret;
 537         const struct lttng_userspace_probe_location *location = nullptr;
 538
 539         LTTNG_ASSERT(ev);
 540         LTTNG_ASSERT(ev->type == LTTNG_EVENT_USERSPACE_PROBE);
 541
 542         location = lttng_event_get_userspace_probe_location(ev);
 543         if (!location) {
 544                 ret = -1;
 545                 goto end;
 546         }
 547
 548         ret = userspace_probe_add_callsite(location, session->uid, session->gid, fd);
 549         if (ret) {
 550                 WARN("Failed to add callsite to userspace probe event '%s'", ev->name);
 551         }
 552
 553 end:
 554         return ret;
 555 }
 556
 557 /*
 558  * Extract the offsets of the instrumentation point for the different look-up
 559  * methods.
 560  */
 561 static int userspace_probe_event_rule_add_callsites(const struct lttng_event_rule *rule,
 562                                                     const struct lttng_credentials *creds,
 563                                                     int fd)
 564 {
 565         int ret;
 566         enum lttng_event_rule_status status;
 567         enum lttng_event_rule_type event_rule_type;
 568         const struct lttng_userspace_probe_location *location = nullptr;
 569
 570         LTTNG_ASSERT(rule);
 571         LTTNG_ASSERT(creds);
 572
 573         event_rule_type = lttng_event_rule_get_type(rule);
 574         LTTNG_ASSERT(event_rule_type == LTTNG_EVENT_RULE_TYPE_KERNEL_UPROBE);
 575
 576         status = lttng_event_rule_kernel_uprobe_get_location(rule, &location);
 577         if (status != LTTNG_EVENT_RULE_STATUS_OK || !location) {
 578                 ret = -1;
 579                 goto end;
 580         }
 581
 582         ret = userspace_probe_add_callsite(
 583                 location, lttng_credentials_get_uid(creds), lttng_credentials_get_gid(creds), fd);
 584         if (ret) {
 585                 WARN("Failed to add callsite to user space probe object: fd = %d", fd);
 586         }
 587
 588 end:
 589         return ret;
 590 }
 591
 592 /*
 593  * Create a kernel event, enable it to the kernel tracer and add it to the
 594  * channel event list of the kernel session.
 595  * We own filter_expression and filter.
 596  */
 597 int kernel_create_event(struct lttng_event *ev,
 598                         struct ltt_kernel_channel *channel,
 599                         char *filter_expression,
 600                         struct lttng_bytecode *filter)
 601 {
 602         int err, fd;
 603         enum lttng_error_code ret;
 604         struct ltt_kernel_event *event;
 605
 606         LTTNG_ASSERT(ev);
 607         LTTNG_ASSERT(channel);
 608
 609         /* We pass ownership of filter_expression and filter */
 610         ret = trace_kernel_create_event(ev, filter_expression, filter, &event);
 611         if (ret != LTTNG_OK) {
 612                 goto error;
 613         }
 614
 615         fd = kernctl_create_event(channel->fd, event->event);
 616         if (fd < 0) {
 617                 switch (-fd) {
 618                 case EEXIST:
 619                         ret = LTTNG_ERR_KERN_EVENT_EXIST;
 620                         break;
 621                 case ENOSYS:
 622                         WARN("Event type not implemented");
 623                         ret = LTTNG_ERR_KERN_EVENT_ENOSYS;
 624                         break;
 625                 case ENOENT:
 626                         WARN("Event %s not found!", ev->name);
 627                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 628                         break;
 629                 default:
 630                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 631                         PERROR("create event ioctl");
 632                 }
 633                 goto free_event;
 634         }
 635
 636         event->type = ev->type;
 637         event->fd = fd;
 638         /* Prevent fd duplication after execlp() */
 639         err = fcntl(event->fd, F_SETFD, FD_CLOEXEC);
 640         if (err < 0) {
 641                 PERROR("fcntl session fd");
 642         }
 643
 644         if (filter) {
 645                 err = kernctl_filter(event->fd, filter);
 646                 if (err < 0) {
 647                         switch (-err) {
 648                         case ENOMEM:
 649                                 ret = LTTNG_ERR_FILTER_NOMEM;
 650                                 break;
 651                         default:
 652                                 ret = LTTNG_ERR_FILTER_INVAL;
 653                                 break;
 654                         }
 655                         goto filter_error;
 656                 }
 657         }
 658
 659         if (ev->type == LTTNG_EVENT_USERSPACE_PROBE) {
 660                 ret = (lttng_error_code) userspace_probe_event_add_callsites(
 661                         ev, channel->session, event->fd);
 662                 if (ret) {
 663                         goto add_callsite_error;
 664                 }
 665         }
 666
 667         err = kernctl_enable(event->fd);
 668         if (err < 0) {
 669                 switch (-err) {
 670                 case EEXIST:
 671                         ret = LTTNG_ERR_KERN_EVENT_EXIST;
 672                         break;
 673                 default:
 674                         PERROR("enable kernel event");
 675                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 676                         break;
 677                 }
 678                 goto enable_error;
 679         }
 680
 681         /* Add event to event list */
 682         cds_list_add(&event->list, &channel->events_list.head);
 683         channel->event_count++;
 684
 685         DBG("Event %s created (fd: %d)", ev->name, event->fd);
 686
 687         return 0;
 688
 689 add_callsite_error:
 690 enable_error:
 691 filter_error:
 692 {
 693         int closeret;
 694
 695         closeret = close(event->fd);
 696         if (closeret) {
 697                 PERROR("close event fd");
 698         }
 699 }
 700 free_event:
 701         free(event);
 702 error:
 703         return ret;
 704 }
 705
 706 /*
 707  * Disable a kernel channel.
 708  */
 709 int kernel_disable_channel(struct ltt_kernel_channel *chan)
 710 {
 711         int ret;
 712
 713         LTTNG_ASSERT(chan);
 714
 715         ret = kernctl_disable(chan->fd);
 716         if (ret < 0) {
 717                 PERROR("disable chan ioctl");
 718                 goto error;
 719         }
 720
 721         chan->enabled = false;
 722         DBG("Kernel channel %s disabled (fd: %d, key: %" PRIu64 ")",
 723             chan->channel->name,
 724             chan->fd,
 725             chan->key);
 726
 727         return 0;
 728
 729 error:
 730         return ret;
 731 }
 732
 733 /*
 734  * Enable a kernel channel.
 735  */
 736 int kernel_enable_channel(struct ltt_kernel_channel *chan)
 737 {
 738         int ret;
 739
 740         LTTNG_ASSERT(chan);
 741
 742         ret = kernctl_enable(chan->fd);
 743         if (ret < 0 && ret != -EEXIST) {
 744                 PERROR("Enable kernel chan");
 745                 goto error;
 746         }
 747
 748         chan->enabled = true;
 749         DBG("Kernel channel %s enabled (fd: %d, key: %" PRIu64 ")",
 750             chan->channel->name,
 751             chan->fd,
 752             chan->key);
 753
 754         return 0;
 755
 756 error:
 757         return ret;
 758 }
 759
 760 /*
 761  * Enable a kernel event.
 762  */
 763 int kernel_enable_event(struct ltt_kernel_event *event)
 764 {
 765         int ret;
 766
 767         LTTNG_ASSERT(event);
 768
 769         ret = kernctl_enable(event->fd);
 770         if (ret < 0) {
 771                 switch (-ret) {
 772                 case EEXIST:
 773                         ret = LTTNG_ERR_KERN_EVENT_EXIST;
 774                         break;
 775                 default:
 776                         PERROR("enable kernel event");
 777                         break;
 778                 }
 779                 goto error;
 780         }
 781
 782         event->enabled = true;
 783         DBG("Kernel event %s enabled (fd: %d)", event->event->name, event->fd);
 784
 785         return 0;
 786
 787 error:
 788         return ret;
 789 }
 790
 791 /*
 792  * Disable a kernel event.
 793  */
 794 int kernel_disable_event(struct ltt_kernel_event *event)
 795 {
 796         int ret;
 797
 798         LTTNG_ASSERT(event);
 799
 800         ret = kernctl_disable(event->fd);
 801         if (ret < 0) {
 802                 PERROR("Failed to disable kernel event: name = '%s', fd = %d",
 803                        event->event->name,
 804                        event->fd);
 805                 goto error;
 806         }
 807
 808         event->enabled = false;
 809         DBG("Kernel event %s disabled (fd: %d)", event->event->name, event->fd);
 810
 811         return 0;
 812
 813 error:
 814         return ret;
 815 }
 816
 817 /*
 818  * Disable a kernel event notifier.
 819  */
 820 static int kernel_disable_event_notifier_rule(struct ltt_kernel_event_notifier_rule *event)
 821 {
 822         int ret;
 823
 824         LTTNG_ASSERT(event);
 825
 826         lttng::urcu::read_lock_guard read_lock;
 827         cds_lfht_del(kernel_token_to_event_notifier_rule_ht, &event->ht_node);
 828
 829         ret = kernctl_disable(event->fd);
 830         if (ret < 0) {
 831                 PERROR("Failed to disable kernel event notifier: fd = %d, token = %" PRIu64,
 832                        event->fd,
 833                        event->token);
 834                 goto error;
 835         }
 836
 837         event->enabled = false;
 838         DBG("Disabled kernel event notifier: fd = %d, token = %" PRIu64, event->fd, event->token);
 839
 840 error:
 841         return ret;
 842 }
 843
 844 static struct process_attr_tracker *
 845 _kernel_get_process_attr_tracker(struct ltt_kernel_session *session,
 846                                  enum lttng_process_attr process_attr)
 847 {
 848         switch (process_attr) {
 849         case LTTNG_PROCESS_ATTR_PROCESS_ID:
 850                 return session->tracker_pid;
 851         case LTTNG_PROCESS_ATTR_VIRTUAL_PROCESS_ID:
 852                 return session->tracker_vpid;
 853         case LTTNG_PROCESS_ATTR_USER_ID:
 854                 return session->tracker_uid;
 855         case LTTNG_PROCESS_ATTR_VIRTUAL_USER_ID:
 856                 return session->tracker_vuid;
 857         case LTTNG_PROCESS_ATTR_GROUP_ID:
 858                 return session->tracker_gid;
 859         case LTTNG_PROCESS_ATTR_VIRTUAL_GROUP_ID:
 860                 return session->tracker_vgid;
 861         default:
 862                 return nullptr;
 863         }
 864 }
 865
 866 const struct process_attr_tracker *
 867 kernel_get_process_attr_tracker(struct ltt_kernel_session *session,
 868                                 enum lttng_process_attr process_attr)
 869 {
 870         return (const struct process_attr_tracker *) _kernel_get_process_attr_tracker(session,
 871                                                                                       process_attr);
 872 }
 873
 874 enum lttng_error_code
 875 kernel_process_attr_tracker_set_tracking_policy(struct ltt_kernel_session *session,
 876                                                 enum lttng_process_attr process_attr,
 877                                                 enum lttng_tracking_policy policy)
 878 {
 879         int ret;
 880         enum lttng_error_code ret_code = LTTNG_OK;
 881         struct process_attr_tracker *tracker =
 882                 _kernel_get_process_attr_tracker(session, process_attr);
 883         enum lttng_tracking_policy previous_policy;
 884
 885         if (!tracker) {
 886                 ret_code = LTTNG_ERR_INVALID;
 887                 goto end;
 888         }
 889
 890         previous_policy = process_attr_tracker_get_tracking_policy(tracker);
 891         ret = process_attr_tracker_set_tracking_policy(tracker, policy);
 892         if (ret) {
 893                 ret_code = LTTNG_ERR_UNK;
 894                 goto end;
 895         }
 896
 897         if (previous_policy == policy) {
 898                 goto end;
 899         }
 900
 901         switch (policy) {
 902         case LTTNG_TRACKING_POLICY_INCLUDE_ALL:
 903                 if (process_attr == LTTNG_PROCESS_ATTR_PROCESS_ID) {
 904                         /*
 905                          * Maintain a special case for the process ID process
 906                          * attribute tracker as it was the only supported
 907                          * attribute prior to 2.12.
 908                          */
 909                         ret = kernctl_track_pid(session->fd, -1);
 910                 } else {
 911                         ret = kernctl_track_id(session->fd, process_attr, -1);
 912                 }
 913                 break;
 914         case LTTNG_TRACKING_POLICY_EXCLUDE_ALL:
 915         case LTTNG_TRACKING_POLICY_INCLUDE_SET:
 916                 /* fall-through. */
 917                 if (process_attr == LTTNG_PROCESS_ATTR_PROCESS_ID) {
 918                         /*
 919                          * Maintain a special case for the process ID process
 920                          * attribute tracker as it was the only supported
 921                          * attribute prior to 2.12.
 922                          */
 923                         ret = kernctl_untrack_pid(session->fd, -1);
 924                 } else {
 925                         ret = kernctl_untrack_id(session->fd, process_attr, -1);
 926                 }
 927                 break;
 928         default:
 929                 abort();
 930         }
 931         /* kern-ctl error handling */
 932         switch (-ret) {
 933         case 0:
 934                 ret_code = LTTNG_OK;
 935                 break;
 936         case EINVAL:
 937                 ret_code = LTTNG_ERR_INVALID;
 938                 break;
 939         case ENOMEM:
 940                 ret_code = LTTNG_ERR_NOMEM;
 941                 break;
 942         case EEXIST:
 943                 ret_code = LTTNG_ERR_PROCESS_ATTR_EXISTS;
 944                 break;
 945         default:
 946                 ret_code = LTTNG_ERR_UNK;
 947                 break;
 948         }
 949 end:
 950         return ret_code;
 951 }
 952
 953 enum lttng_error_code
 954 kernel_process_attr_tracker_inclusion_set_add_value(struct ltt_kernel_session *session,
 955                                                     enum lttng_process_attr process_attr,
 956                                                     const struct process_attr_value *value)
 957 {
 958         int ret, integral_value;
 959         enum lttng_error_code ret_code;
 960         struct process_attr_tracker *tracker;
 961         enum process_attr_tracker_status status;
 962
 963         /*
 964          * Convert process attribute tracker value to the integral
 965          * representation required by the kern-ctl API.
 966          */
 967         switch (process_attr) {
 968         case LTTNG_PROCESS_ATTR_PROCESS_ID:
 969         case LTTNG_PROCESS_ATTR_VIRTUAL_PROCESS_ID:
 970                 integral_value = (int) value->value.pid;
 971                 break;
 972         case LTTNG_PROCESS_ATTR_USER_ID:
 973         case LTTNG_PROCESS_ATTR_VIRTUAL_USER_ID:
 974                 if (value->type == LTTNG_PROCESS_ATTR_VALUE_TYPE_USER_NAME) {
 975                         uid_t uid;
 976
 977                         ret_code = utils_user_id_from_name(value->value.user_name, &uid);
 978                         if (ret_code != LTTNG_OK) {
 979                                 goto end;
 980                         }
 981                         integral_value = (int) uid;
 982                 } else {
 983                         integral_value = (int) value->value.uid;
 984                 }
 985                 break;
 986         case LTTNG_PROCESS_ATTR_GROUP_ID:
 987         case LTTNG_PROCESS_ATTR_VIRTUAL_GROUP_ID:
 988                 if (value->type == LTTNG_PROCESS_ATTR_VALUE_TYPE_GROUP_NAME) {
 989                         gid_t gid;
 990
 991                         ret_code = utils_group_id_from_name(value->value.group_name, &gid);
 992                         if (ret_code != LTTNG_OK) {
 993                                 goto end;
 994                         }
 995                         integral_value = (int) gid;
 996                 } else {
 997                         integral_value = (int) value->value.gid;
 998                 }
 999                 break;
1000         default:
1001                 ret_code = LTTNG_ERR_INVALID;
1002                 goto end;
1003         }
1004
1005         tracker = _kernel_get_process_attr_tracker(session, process_attr);
1006         if (!tracker) {
1007                 ret_code = LTTNG_ERR_INVALID;
1008                 goto end;
1009         }
1010
1011         status = process_attr_tracker_inclusion_set_add_value(tracker, value);
1012         if (status != PROCESS_ATTR_TRACKER_STATUS_OK) {
1013                 switch (status) {
1014                 case PROCESS_ATTR_TRACKER_STATUS_EXISTS:
1015                         ret_code = LTTNG_ERR_PROCESS_ATTR_EXISTS;
1016                         break;
1017                 case PROCESS_ATTR_TRACKER_STATUS_INVALID_TRACKING_POLICY:
1018                         ret_code = LTTNG_ERR_PROCESS_ATTR_TRACKER_INVALID_TRACKING_POLICY;
1019                         break;
1020                 case PROCESS_ATTR_TRACKER_STATUS_ERROR:
1021                 default:
1022                         ret_code = LTTNG_ERR_UNK;
1023                         break;
1024                 }
1025                 goto end;
1026         }
1027
1028         DBG("Kernel track %s %d for session id %" PRIu64,
1029             lttng_process_attr_to_string(process_attr),
1030             integral_value,
1031             session->id);
1032         if (process_attr == LTTNG_PROCESS_ATTR_PROCESS_ID) {
1033                 /*
1034                  * Maintain a special case for the process ID process attribute
1035                  * tracker as it was the only supported attribute prior to 2.12.
1036                  */
1037                 ret = kernctl_track_pid(session->fd, integral_value);
1038         } else {
1039                 ret = kernctl_track_id(session->fd, process_attr, integral_value);
1040         }
1041         if (ret == 0) {
1042                 ret_code = LTTNG_OK;
1043                 goto end;
1044         }
1045
1046         kernel_wait_quiescent();
1047
1048         /* kern-ctl error handling */
1049         switch (-ret) {
1050         case 0:
1051                 ret_code = LTTNG_OK;
1052                 break;
1053         case EINVAL:
1054                 ret_code = LTTNG_ERR_INVALID;
1055                 break;
1056         case ENOMEM:
1057                 ret_code = LTTNG_ERR_NOMEM;
1058                 break;
1059         case EEXIST:
1060                 ret_code = LTTNG_ERR_PROCESS_ATTR_EXISTS;
1061                 break;
1062         default:
1063                 ret_code = LTTNG_ERR_UNK;
1064                 break;
1065         }
1066
1067         /* Attempt to remove the value from the tracker. */
1068         status = process_attr_tracker_inclusion_set_remove_value(tracker, value);
1069         if (status != PROCESS_ATTR_TRACKER_STATUS_OK) {
1070                 ERR("Failed to roll-back the tracking of kernel %s process attribute %d while handling a kern-ctl error",
1071                     lttng_process_attr_to_string(process_attr),
1072                     integral_value);
1073         }
1074 end:
1075         return ret_code;
1076 }
1077
1078 enum lttng_error_code
1079 kernel_process_attr_tracker_inclusion_set_remove_value(struct ltt_kernel_session *session,
1080                                                        enum lttng_process_attr process_attr,
1081                                                        const struct process_attr_value *value)
1082 {
1083         int ret, integral_value;
1084         enum lttng_error_code ret_code;
1085         struct process_attr_tracker *tracker;
1086         enum process_attr_tracker_status status;
1087
1088         /*
1089          * Convert process attribute tracker value to the integral
1090          * representation required by the kern-ctl API.
1091          */
1092         switch (process_attr) {
1093         case LTTNG_PROCESS_ATTR_PROCESS_ID:
1094         case LTTNG_PROCESS_ATTR_VIRTUAL_PROCESS_ID:
1095                 integral_value = (int) value->value.pid;
1096                 break;
1097         case LTTNG_PROCESS_ATTR_USER_ID:
1098         case LTTNG_PROCESS_ATTR_VIRTUAL_USER_ID:
1099                 if (value->type == LTTNG_PROCESS_ATTR_VALUE_TYPE_USER_NAME) {
1100                         uid_t uid;
1101
1102                         ret_code = utils_user_id_from_name(value->value.user_name, &uid);
1103                         if (ret_code != LTTNG_OK) {
1104                                 goto end;
1105                         }
1106                         integral_value = (int) uid;
1107                 } else {
1108                         integral_value = (int) value->value.uid;
1109                 }
1110                 break;
1111         case LTTNG_PROCESS_ATTR_GROUP_ID:
1112         case LTTNG_PROCESS_ATTR_VIRTUAL_GROUP_ID:
1113                 if (value->type == LTTNG_PROCESS_ATTR_VALUE_TYPE_GROUP_NAME) {
1114                         gid_t gid;
1115
1116                         ret_code = utils_group_id_from_name(value->value.group_name, &gid);
1117                         if (ret_code != LTTNG_OK) {
1118                                 goto end;
1119                         }
1120                         integral_value = (int) gid;
1121                 } else {
1122                         integral_value = (int) value->value.gid;
1123                 }
1124                 break;
1125         default:
1126                 ret_code = LTTNG_ERR_INVALID;
1127                 goto end;
1128         }
1129
1130         tracker = _kernel_get_process_attr_tracker(session, process_attr);
1131         if (!tracker) {
1132                 ret_code = LTTNG_ERR_INVALID;
1133                 goto end;
1134         }
1135
1136         status = process_attr_tracker_inclusion_set_remove_value(tracker, value);
1137         if (status != PROCESS_ATTR_TRACKER_STATUS_OK) {
1138                 switch (status) {
1139                 case PROCESS_ATTR_TRACKER_STATUS_MISSING:
1140                         ret_code = LTTNG_ERR_PROCESS_ATTR_MISSING;
1141                         break;
1142                 case PROCESS_ATTR_TRACKER_STATUS_INVALID_TRACKING_POLICY:
1143                         ret_code = LTTNG_ERR_PROCESS_ATTR_TRACKER_INVALID_TRACKING_POLICY;
1144                         break;
1145                 case PROCESS_ATTR_TRACKER_STATUS_ERROR:
1146                 default:
1147                         ret_code = LTTNG_ERR_UNK;
1148                         break;
1149                 }
1150                 goto end;
1151         }
1152
1153         DBG("Kernel track %s %d for session id %" PRIu64,
1154             lttng_process_attr_to_string(process_attr),
1155             integral_value,
1156             session->id);
1157         if (process_attr == LTTNG_PROCESS_ATTR_PROCESS_ID) {
1158                 /*
1159                  * Maintain a special case for the process ID process attribute
1160                  * tracker as it was the only supported attribute prior to 2.12.
1161                  */
1162                 ret = kernctl_untrack_pid(session->fd, integral_value);
1163         } else {
1164                 ret = kernctl_untrack_id(session->fd, process_attr, integral_value);
1165         }
1166         if (ret == 0) {
1167                 ret_code = LTTNG_OK;
1168                 goto end;
1169         }
1170         kernel_wait_quiescent();
1171
1172         /* kern-ctl error handling */
1173         switch (-ret) {
1174         case 0:
1175                 ret_code = LTTNG_OK;
1176                 break;
1177         case EINVAL:
1178                 ret_code = LTTNG_ERR_INVALID;
1179                 break;
1180         case ENOMEM:
1181                 ret_code = LTTNG_ERR_NOMEM;
1182                 break;
1183         case ENOENT:
1184                 ret_code = LTTNG_ERR_PROCESS_ATTR_MISSING;
1185                 break;
1186         default:
1187                 ret_code = LTTNG_ERR_UNK;
1188                 break;
1189         }
1190
1191         /* Attempt to add the value to the tracker. */
1192         status = process_attr_tracker_inclusion_set_add_value(tracker, value);
1193         if (status != PROCESS_ATTR_TRACKER_STATUS_OK) {
1194                 ERR("Failed to roll-back the tracking of kernel %s process attribute %d while handling a kern-ctl error",
1195                     lttng_process_attr_to_string(process_attr),
1196                     integral_value);
1197         }
1198 end:
1199         return ret_code;
1200 }
1201
1202 /*
1203  * Create kernel metadata, open from the kernel tracer and add it to the
1204  * kernel session.
1205  */
1206 int kernel_open_metadata(struct ltt_kernel_session *session)
1207 {
1208         int ret;
1209         struct ltt_kernel_metadata *lkm = nullptr;
1210
1211         LTTNG_ASSERT(session);
1212
1213         /* Allocate kernel metadata */
1214         lkm = trace_kernel_create_metadata();
1215         if (lkm == nullptr) {
1216                 goto error;
1217         }
1218
1219         /* Kernel tracer metadata creation */
1220         ret = kernctl_open_metadata(session->fd, &lkm->conf->attr);
1221         if (ret < 0) {
1222                 goto error_open;
1223         }
1224
1225         lkm->fd = ret;
1226         lkm->key = ++next_kernel_channel_key;
1227         /* Prevent fd duplication after execlp() */
1228         ret = fcntl(lkm->fd, F_SETFD, FD_CLOEXEC);
1229         if (ret < 0) {
1230                 PERROR("fcntl session fd");
1231         }
1232
1233         session->metadata = lkm;
1234
1235         DBG("Kernel metadata opened (fd: %d)", lkm->fd);
1236
1237         return 0;
1238
1239 error_open:
1240         trace_kernel_destroy_metadata(lkm);
1241 error:
1242         return -1;
1243 }
1244
1245 /*
1246  * Start tracing session.
1247  */
1248 int kernel_start_session(struct ltt_kernel_session *session)
1249 {
1250         int ret;
1251
1252         LTTNG_ASSERT(session);
1253
1254         ret = kernctl_start_session(session->fd);
1255         if (ret < 0) {
1256                 PERROR("ioctl start session");
1257                 goto error;
1258         }
1259
1260         DBG("Kernel session started");
1261
1262         return 0;
1263
1264 error:
1265         return ret;
1266 }
1267
1268 /*
1269  * Make a kernel wait to make sure in-flight probe have completed.
1270  */
1271 void kernel_wait_quiescent()
1272 {
1273         int ret;
1274         int fd = kernel_tracer_fd;
1275
1276         DBG("Kernel quiescent wait on %d", fd);
1277
1278         ret = kernctl_wait_quiescent(fd);
1279         if (ret < 0) {
1280                 PERROR("wait quiescent ioctl");
1281                 ERR("Kernel quiescent wait failed");
1282         }
1283 }
1284
1285 /*
1286  *  Force flush buffer of metadata.
1287  */
1288 int kernel_metadata_flush_buffer(int fd)
1289 {
1290         int ret;
1291
1292         DBG("Kernel flushing metadata buffer on fd %d", fd);
1293
1294         ret = kernctl_buffer_flush(fd);
1295         if (ret < 0) {
1296                 ERR("Fail to flush metadata buffers %d (ret: %d)", fd, ret);
1297         }
1298
1299         return 0;
1300 }
1301
1302 /*
1303  * Force flush buffer for channel.
1304  */
1305 int kernel_flush_buffer(struct ltt_kernel_channel *channel)
1306 {
1307         int ret;
1308         struct ltt_kernel_stream *stream;
1309
1310         LTTNG_ASSERT(channel);
1311
1312         DBG("Flush buffer for channel %s", channel->channel->name);
1313
1314         cds_list_for_each_entry (stream, &channel->stream_list.head, list) {
1315                 DBG("Flushing channel stream %d", stream->fd);
1316                 ret = kernctl_buffer_flush(stream->fd);
1317                 if (ret < 0) {
1318                         PERROR("ioctl");
1319                         ERR("Fail to flush buffer for stream %d (ret: %d)", stream->fd, ret);
1320                 }
1321         }
1322
1323         return 0;
1324 }
1325
1326 /*
1327  * Stop tracing session.
1328  */
1329 int kernel_stop_session(struct ltt_kernel_session *session)
1330 {
1331         int ret;
1332
1333         LTTNG_ASSERT(session);
1334
1335         ret = kernctl_stop_session(session->fd);
1336         if (ret < 0) {
1337                 goto error;
1338         }
1339
1340         DBG("Kernel session stopped");
1341
1342         return 0;
1343
1344 error:
1345         return ret;
1346 }
1347
1348 /*
1349  * Open stream of channel, register it to the kernel tracer and add it
1350  * to the stream list of the channel.
1351  *
1352  * Note: given that the streams may appear in random order wrt CPU
1353  * number (e.g. cpu hotplug), the index value of the stream number in
1354  * the stream name is not necessarily linked to the CPU number.
1355  *
1356  * Return the number of created stream. Else, a negative value.
1357  */
1358 int kernel_open_channel_stream(struct ltt_kernel_channel *channel)
1359 {
1360         int ret;
1361         struct ltt_kernel_stream *lks;
1362
1363         LTTNG_ASSERT(channel);
1364
1365         while ((ret = kernctl_create_stream(channel->fd)) >= 0) {
1366                 lks = trace_kernel_create_stream(channel->channel->name, channel->stream_count);
1367                 if (lks == nullptr) {
1368                         ret = close(ret);
1369                         if (ret) {
1370                                 PERROR("close");
1371                         }
1372                         goto error;
1373                 }
1374
1375                 lks->fd = ret;
1376                 /* Prevent fd duplication after execlp() */
1377                 ret = fcntl(lks->fd, F_SETFD, FD_CLOEXEC);
1378                 if (ret < 0) {
1379                         PERROR("fcntl session fd");
1380                 }
1381
1382                 lks->tracefile_size = channel->channel->attr.tracefile_size;
1383                 lks->tracefile_count = channel->channel->attr.tracefile_count;
1384
1385                 /* Add stream to channel stream list */
1386                 cds_list_add(&lks->list, &channel->stream_list.head);
1387                 channel->stream_count++;
1388
1389                 DBG("Kernel stream %s created (fd: %d, state: %d)", lks->name, lks->fd, lks->state);
1390         }
1391
1392         return channel->stream_count;
1393
1394 error:
1395         return -1;
1396 }
1397
1398 /*
1399  * Open the metadata stream and set it to the kernel session.
1400  */
1401 int kernel_open_metadata_stream(struct ltt_kernel_session *session)
1402 {
1403         int ret;
1404
1405         LTTNG_ASSERT(session);
1406
1407         ret = kernctl_create_stream(session->metadata->fd);
1408         if (ret < 0) {
1409                 PERROR("kernel create metadata stream");
1410                 goto error;
1411         }
1412
1413         DBG("Kernel metadata stream created (fd: %d)", ret);
1414         session->metadata_stream_fd = ret;
1415         /* Prevent fd duplication after execlp() */
1416         ret = fcntl(session->metadata_stream_fd, F_SETFD, FD_CLOEXEC);
1417         if (ret < 0) {
1418                 PERROR("fcntl session fd");
1419         }
1420
1421         return 0;
1422
1423 error:
1424         return -1;
1425 }
1426
1427 /*
1428  * Get the event list from the kernel tracer and return the number of elements.
1429  */
1430 ssize_t kernel_list_events(struct lttng_event **events)
1431 {
1432         int fd, ret;
1433         char *event;
1434         size_t nbmem, count = 0;
1435         FILE *fp;
1436         struct lttng_event *elist;
1437
1438         LTTNG_ASSERT(events);
1439
1440         fd = kernctl_tracepoint_list(kernel_tracer_fd);
1441         if (fd < 0) {
1442                 PERROR("kernel tracepoint list");
1443                 goto error;
1444         }
1445
1446         fp = fdopen(fd, "r");
1447         if (fp == nullptr) {
1448                 PERROR("kernel tracepoint list fdopen");
1449                 goto error_fp;
1450         }
1451
1452         /*
1453          * Init memory size counter
1454          * See kernel-ctl.h for explanation of this value
1455          */
1456         nbmem = KERNEL_EVENT_INIT_LIST_SIZE;
1457         elist = calloc<lttng_event>(nbmem);
1458         if (elist == nullptr) {
1459                 PERROR("alloc list events");
1460                 count = -ENOMEM;
1461                 goto end;
1462         }
1463
1464         while (fscanf(fp, "event { name = %m[^;]; };\n", &event) == 1) {
1465                 if (count >= nbmem) {
1466                         struct lttng_event *new_elist;
1467                         size_t new_nbmem;
1468
1469                         new_nbmem = nbmem << 1;
1470                         DBG("Reallocating event list from %zu to %zu bytes", nbmem, new_nbmem);
1471                         new_elist = (lttng_event *) realloc(elist,
1472                                                             new_nbmem * sizeof(struct lttng_event));
1473                         if (new_elist == nullptr) {
1474                                 PERROR("realloc list events");
1475                                 free(event);
1476                                 free(elist);
1477                                 count = -ENOMEM;
1478                                 goto end;
1479                         }
1480                         /* Zero the new memory */
1481                         memset(new_elist + nbmem,
1482                                0,
1483                                (new_nbmem - nbmem) * sizeof(struct lttng_event));
1484                         nbmem = new_nbmem;
1485                         elist = new_elist;
1486                 }
1487                 strncpy(elist[count].name, event, LTTNG_SYMBOL_NAME_LEN);
1488                 elist[count].name[LTTNG_SYMBOL_NAME_LEN - 1] = '\0';
1489                 elist[count].enabled = -1;
1490                 count++;
1491                 free(event);
1492         }
1493
1494         *events = elist;
1495         DBG("Kernel list events done (%zu events)", count);
1496 end:
1497         ret = fclose(fp); /* closes both fp and fd */
1498         if (ret) {
1499                 PERROR("fclose");
1500         }
1501         return count;
1502
1503 error_fp:
1504         ret = close(fd);
1505         if (ret) {
1506                 PERROR("close");
1507         }
1508 error:
1509         return -1;
1510 }
1511
1512 /*
1513  * Get kernel version and validate it.
1514  */
1515 int kernel_validate_version(struct lttng_kernel_abi_tracer_version *version,
1516                             struct lttng_kernel_abi_tracer_abi_version *abi_version)
1517 {
1518         int ret;
1519
1520         ret = kernctl_tracer_version(kernel_tracer_fd, version);
1521         if (ret < 0) {
1522                 ERR("Failed to retrieve the lttng-modules version");
1523                 goto error;
1524         }
1525
1526         /* Validate version */
1527         if (version->major != VERSION_MAJOR) {
1528                 ERR("Kernel tracer major version (%d) is not compatible with lttng-tools major version (%d)",
1529                     version->major,
1530                     VERSION_MAJOR);
1531                 goto error_version;
1532         }
1533         ret = kernctl_tracer_abi_version(kernel_tracer_fd, abi_version);
1534         if (ret < 0) {
1535                 ERR("Failed to retrieve lttng-modules ABI version");
1536                 goto error;
1537         }
1538         if (abi_version->major != LTTNG_KERNEL_ABI_MAJOR_VERSION) {
1539                 ERR("Kernel tracer ABI version (%d.%d) does not match the expected ABI major version (%d.*)",
1540                     abi_version->major,
1541                     abi_version->minor,
1542                     LTTNG_KERNEL_ABI_MAJOR_VERSION);
1543                 goto error;
1544         }
1545         DBG2("Kernel tracer version validated (%d.%d, ABI %d.%d)",
1546              version->major,
1547              version->minor,
1548              abi_version->major,
1549              abi_version->minor);
1550         return 0;
1551
1552 error_version:
1553         ret = -1;
1554
1555 error:
1556         ERR("Kernel tracer version check failed; kernel tracing will not be available");
1557         return ret;
1558 }
1559
1560 /*
1561  * Kernel work-arounds called at the start of sessiond main().
1562  */
1563 int init_kernel_workarounds()
1564 {
1565         int ret;
1566         FILE *fp;
1567
1568         /*
1569          * boot_id needs to be read once before being used concurrently
1570          * to deal with a Linux kernel race. A fix is proposed for
1571          * upstream, but the work-around is needed for older kernels.
1572          */
1573         fp = fopen("/proc/sys/kernel/random/boot_id", "r");
1574         if (!fp) {
1575                 goto end_boot_id;
1576         }
1577         while (!feof(fp)) {
1578                 char buf[37] = "";
1579
1580                 ret = fread(buf, 1, sizeof(buf), fp);
1581                 if (ret < 0) {
1582                         /* Ignore error, we don't really care */
1583                 }
1584         }
1585         ret = fclose(fp);
1586         if (ret) {
1587                 PERROR("fclose");
1588         }
1589 end_boot_id:
1590         return 0;
1591 }
1592
1593 /*
1594  * Teardown of a kernel session, keeping data required by destroy notifiers.
1595  */
1596 void kernel_destroy_session(struct ltt_kernel_session *ksess)
1597 {
1598         struct lttng_trace_chunk *trace_chunk;
1599
1600         if (ksess == nullptr) {
1601                 DBG3("No kernel session when tearing down session");
1602                 return;
1603         }
1604
1605         DBG("Tearing down kernel session");
1606         trace_chunk = ksess->current_trace_chunk;
1607
1608         /*
1609          * Destroy channels on the consumer if at least one FD has been sent and we
1610          * are in no output mode because the streams are in *no* monitor mode so we
1611          * have to send a command to clean them up or else they leaked.
1612          */
1613         if (!ksess->output_traces && ksess->consumer_fds_sent) {
1614                 int ret;
1615                 struct consumer_socket *socket;
1616                 struct lttng_ht_iter iter;
1617
1618                 /* For each consumer socket. */
1619                 lttng::urcu::read_lock_guard read_lock;
1620
1621                 cds_lfht_for_each_entry (
1622                         ksess->consumer->socks->ht, &iter.iter, socket, node.node) {
1623                         struct ltt_kernel_channel *chan;
1624
1625                         /* For each channel, ask the consumer to destroy it. */
1626                         cds_list_for_each_entry (chan, &ksess->channel_list.head, list) {
1627                                 ret = kernel_consumer_destroy_channel(socket, chan);
1628                                 if (ret < 0) {
1629                                         /* Consumer is probably dead. Use next socket. */
1630                                         continue;
1631                                 }
1632                         }
1633                 }
1634         }
1635
1636         /* Close any relayd session */
1637         consumer_output_send_destroy_relayd(ksess->consumer);
1638
1639         trace_kernel_destroy_session(ksess);
1640         lttng_trace_chunk_put(trace_chunk);
1641 }
1642
1643 /* Teardown of data required by destroy notifiers. */
1644 void kernel_free_session(struct ltt_kernel_session *ksess)
1645 {
1646         if (ksess == nullptr) {
1647                 return;
1648         }
1649         trace_kernel_free_session(ksess);
1650 }
1651
1652 /*
1653  * Destroy a kernel channel object. It does not do anything on the tracer side.
1654  */
1655 void kernel_destroy_channel(struct ltt_kernel_channel *kchan)
1656 {
1657         struct ltt_kernel_session *ksess = nullptr;
1658
1659         LTTNG_ASSERT(kchan);
1660         LTTNG_ASSERT(kchan->channel);
1661
1662         DBG3("Kernel destroy channel %s", kchan->channel->name);
1663
1664         /* Update channel count of associated session. */
1665         if (kchan->session) {
1666                 /* Keep pointer reference so we can update it after the destroy. */
1667                 ksess = kchan->session;
1668         }
1669
1670         trace_kernel_destroy_channel(kchan);
1671
1672         /*
1673          * At this point the kernel channel is not visible anymore. This is safe
1674          * since in order to work on a visible kernel session, the tracing session
1675          * lock (ltt_session.lock) MUST be acquired.
1676          */
1677         if (ksess) {
1678                 ksess->channel_count--;
1679         }
1680 }
1681
1682 /*
1683  * Take a snapshot for a given kernel session.
1684  *
1685  * Return LTTNG_OK on success or else return a LTTNG_ERR code.
1686  */
1687 enum lttng_error_code kernel_snapshot_record(struct ltt_kernel_session *ksess,
1688                                              const struct consumer_output *output,
1689                                              uint64_t nb_packets_per_stream)
1690 {
1691         int err, ret, saved_metadata_fd;
1692         enum lttng_error_code status = LTTNG_OK;
1693         struct consumer_socket *socket;
1694         struct lttng_ht_iter iter;
1695         struct ltt_kernel_metadata *saved_metadata;
1696         char *trace_path = nullptr;
1697         size_t consumer_path_offset = 0;
1698
1699         LTTNG_ASSERT(ksess);
1700         LTTNG_ASSERT(ksess->consumer);
1701         LTTNG_ASSERT(output);
1702
1703         DBG("Kernel snapshot record started");
1704
1705         /* Save current metadata since the following calls will change it. */
1706         saved_metadata = ksess->metadata;
1707         saved_metadata_fd = ksess->metadata_stream_fd;
1708
1709         ret = kernel_open_metadata(ksess);
1710         if (ret < 0) {
1711                 status = LTTNG_ERR_KERN_META_FAIL;
1712                 goto error;
1713         }
1714
1715         ret = kernel_open_metadata_stream(ksess);
1716         if (ret < 0) {
1717                 status = LTTNG_ERR_KERN_META_FAIL;
1718                 goto error_open_stream;
1719         }
1720
1721         trace_path = setup_channel_trace_path(ksess->consumer, "", &consumer_path_offset);
1722         if (!trace_path) {
1723                 status = LTTNG_ERR_INVALID;
1724                 goto error;
1725         }
1726
1727         {
1728                 /* Send metadata to consumer and snapshot everything. */
1729                 lttng::urcu::read_lock_guard read_lock;
1730
1731                 cds_lfht_for_each_entry (output->socks->ht, &iter.iter, socket, node.node) {
1732                         struct ltt_kernel_channel *chan;
1733
1734                         pthread_mutex_lock(socket->lock);
1735                         /* This stream must not be monitored by the consumer. */
1736                         ret = kernel_consumer_add_metadata(socket, ksess, 0);
1737                         pthread_mutex_unlock(socket->lock);
1738                         if (ret < 0) {
1739                                 status = LTTNG_ERR_KERN_META_FAIL;
1740                                 goto error_consumer;
1741                         }
1742
1743                         /* For each channel, ask the consumer to snapshot it. */
1744                         cds_list_for_each_entry (chan, &ksess->channel_list.head, list) {
1745                                 status =
1746                                         consumer_snapshot_channel(socket,
1747                                                                   chan->key,
1748                                                                   output,
1749                                                                   0,
1750                                                                   &trace_path[consumer_path_offset],
1751                                                                   nb_packets_per_stream);
1752                                 if (status != LTTNG_OK) {
1753                                         (void) kernel_consumer_destroy_metadata(socket,
1754                                                                                 ksess->metadata);
1755                                         goto error_consumer;
1756                                 }
1757                         }
1758
1759                         /* Snapshot metadata, */
1760                         status = consumer_snapshot_channel(socket,
1761                                                            ksess->metadata->key,
1762                                                            output,
1763                                                            1,
1764                                                            &trace_path[consumer_path_offset],
1765                                                            0);
1766                         if (status != LTTNG_OK) {
1767                                 goto error_consumer;
1768                         }
1769
1770                         /*
1771                          * The metadata snapshot is done, ask the consumer to destroy it since
1772                          * it's not monitored on the consumer side.
1773                          */
1774                         (void) kernel_consumer_destroy_metadata(socket, ksess->metadata);
1775                 }
1776         }
1777
1778 error_consumer:
1779         /* Close newly opened metadata stream. It's now on the consumer side. */
1780         err = close(ksess->metadata_stream_fd);
1781         if (err < 0) {
1782                 PERROR("close snapshot kernel");
1783         }
1784
1785 error_open_stream:
1786         trace_kernel_destroy_metadata(ksess->metadata);
1787 error:
1788         /* Restore metadata state.*/
1789         ksess->metadata = saved_metadata;
1790         ksess->metadata_stream_fd = saved_metadata_fd;
1791         free(trace_path);
1792         return status;
1793 }
1794
1795 /*
1796  * Get the syscall mask array from the kernel tracer.
1797  *
1798  * Return 0 on success else a negative value. In both case, syscall_mask should
1799  * be freed.
1800  */
1801 int kernel_syscall_mask(int chan_fd, char **syscall_mask, uint32_t *nr_bits)
1802 {
1803         LTTNG_ASSERT(syscall_mask);
1804         LTTNG_ASSERT(nr_bits);
1805
1806         return kernctl_syscall_mask(chan_fd, syscall_mask, nr_bits);
1807 }
1808
1809 static int kernel_tracer_abi_greater_or_equal(unsigned int major, unsigned int minor)
1810 {
1811         int ret;
1812         struct lttng_kernel_abi_tracer_abi_version abi;
1813
1814         ret = kernctl_tracer_abi_version(kernel_tracer_fd, &abi);
1815         if (ret < 0) {
1816                 ERR("Failed to retrieve lttng-modules ABI version");
1817                 goto error;
1818         }
1819
1820         ret = abi.major > major || (abi.major == major && abi.minor >= minor);
1821 error:
1822         return ret;
1823 }
1824
1825 /*
1826  * Check for the support of the RING_BUFFER_SNAPSHOT_SAMPLE_POSITIONS via abi
1827  * version number.
1828  *
1829  * Return 1 on success, 0 when feature is not supported, negative value in case
1830  * of errors.
1831  */
1832 int kernel_supports_ring_buffer_snapshot_sample_positions()
1833 {
1834         /*
1835          * RING_BUFFER_SNAPSHOT_SAMPLE_POSITIONS was introduced in 2.3
1836          */
1837         return kernel_tracer_abi_greater_or_equal(2, 3);
1838 }
1839
1840 /*
1841  * Check for the support of the packet sequence number via abi version number.
1842  *
1843  * Return 1 on success, 0 when feature is not supported, negative value in case
1844  * of errors.
1845  */
1846 int kernel_supports_ring_buffer_packet_sequence_number()
1847 {
1848         /*
1849          * Packet sequence number was introduced in LTTng 2.8,
1850          * lttng-modules ABI 2.1.
1851          */
1852         return kernel_tracer_abi_greater_or_equal(2, 1);
1853 }
1854
1855 /*
1856  * Check for the support of event notifiers via abi version number.
1857  *
1858  * Return 1 on success, 0 when feature is not supported, negative value in case
1859  * of errors.
1860  */
1861 int kernel_supports_event_notifiers()
1862 {
1863         /*
1864          * Event notifiers were introduced in LTTng 2.13, lttng-modules ABI 2.6.
1865          */
1866         return kernel_tracer_abi_greater_or_equal(2, 6);
1867 }
1868
1869 /*
1870  * Rotate a kernel session.
1871  *
1872  * Return LTTNG_OK on success or else an LTTng error code.
1873  */
1874 enum lttng_error_code kernel_rotate_session(struct ltt_session *session)
1875 {
1876         int ret;
1877         enum lttng_error_code status = LTTNG_OK;
1878         struct consumer_socket *socket;
1879         struct lttng_ht_iter iter;
1880         struct ltt_kernel_session *ksess = session->kernel_session;
1881
1882         LTTNG_ASSERT(ksess);
1883         LTTNG_ASSERT(ksess->consumer);
1884
1885         DBG("Rotate kernel session %s started (session %" PRIu64 ")", session->name, session->id);
1886
1887         {
1888                 /*
1889                  * Note that this loop will end after one iteration given that there is
1890                  * only one kernel consumer.
1891                  */
1892                 lttng::urcu::read_lock_guard read_lock;
1893
1894                 cds_lfht_for_each_entry (
1895                         ksess->consumer->socks->ht, &iter.iter, socket, node.node) {
1896                         struct ltt_kernel_channel *chan;
1897
1898                         /* For each channel, ask the consumer to rotate it. */
1899                         cds_list_for_each_entry (chan, &ksess->channel_list.head, list) {
1900                                 DBG("Rotate kernel channel %" PRIu64 ", session %s",
1901                                     chan->key,
1902                                     session->name);
1903                                 ret = consumer_rotate_channel(socket,
1904                                                               chan->key,
1905                                                               ksess->consumer,
1906                                                               /* is_metadata_channel */ false);
1907                                 if (ret < 0) {
1908                                         status = LTTNG_ERR_ROTATION_FAIL_CONSUMER;
1909                                         goto error;
1910                                 }
1911                         }
1912
1913                         /*
1914                          * Rotate the metadata channel.
1915                          */
1916                         ret = consumer_rotate_channel(socket,
1917                                                       ksess->metadata->key,
1918                                                       ksess->consumer,
1919                                                       /* is_metadata_channel */ true);
1920                         if (ret < 0) {
1921                                 status = LTTNG_ERR_ROTATION_FAIL_CONSUMER;
1922                                 goto error;
1923                         }
1924                 }
1925         }
1926
1927 error:
1928         return status;
1929 }
1930
1931 enum lttng_error_code kernel_create_channel_subdirectories(const struct ltt_kernel_session *ksess)
1932 {
1933         enum lttng_error_code ret = LTTNG_OK;
1934         enum lttng_trace_chunk_status chunk_status;
1935
1936         lttng::urcu::read_lock_guard read_lock;
1937         LTTNG_ASSERT(ksess->current_trace_chunk);
1938
1939         /*
1940          * Create the index subdirectory which will take care
1941          * of implicitly creating the channel's path.
1942          */
1943         chunk_status = lttng_trace_chunk_create_subdirectory(
1944                 ksess->current_trace_chunk, DEFAULT_KERNEL_TRACE_DIR "/" DEFAULT_INDEX_DIR);
1945         if (chunk_status != LTTNG_TRACE_CHUNK_STATUS_OK) {
1946                 ret = LTTNG_ERR_CREATE_DIR_FAIL;
1947                 goto error;
1948         }
1949 error:
1950         return ret;
1951 }
1952
1953 /*
1954  * Get current kernel tracer status
1955  */
1956 enum lttng_kernel_tracer_status get_kernel_tracer_status()
1957 {
1958         if (!kernel_tracer_status) {
1959                 return LTTNG_KERNEL_TRACER_STATUS_ERR_UNKNOWN;
1960         }
1961
1962         return *kernel_tracer_status;
1963 }
1964
1965 /*
1966  * Sets the kernel tracer status based on the positive errno code
1967  */
1968 void set_kernel_tracer_status_from_modules_ret(int code)
1969 {
1970         switch (code) {
1971         case ENOENT:
1972         {
1973                 kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
1974                         LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_MISSING);
1975                 break;
1976         }
1977         case ENOKEY:
1978         case EKEYEXPIRED:
1979         case EKEYREVOKED:
1980         case EKEYREJECTED:
1981         {
1982                 kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
1983                         LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_SIGNATURE);
1984                 break;
1985         }
1986         default:
1987         {
1988                 kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
1989                         LTTNG_KERNEL_TRACER_STATUS_ERR_MODULES_UNKNOWN);
1990                 break;
1991         }
1992         }
1993 }
1994
1995 /*
1996  * Setup necessary data for kernel tracer action.
1997  */
1998 int init_kernel_tracer()
1999 {
2000         int ret;
2001         bool is_root = !getuid();
2002
2003         /* Modprobe lttng kernel modules */
2004         ret = modprobe_lttng_control();
2005         if (ret < 0) {
2006                 set_kernel_tracer_status_from_modules_ret(-ret);
2007                 goto error;
2008         }
2009
2010         /* Open debugfs lttng */
2011         kernel_tracer_fd = open(module_proc_lttng, O_RDWR);
2012         if (kernel_tracer_fd < 0) {
2013                 DBG("Failed to open %s", module_proc_lttng);
2014                 kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
2015                         LTTNG_KERNEL_TRACER_STATUS_ERR_OPEN_PROC_LTTNG);
2016                 goto error_open;
2017         }
2018
2019         /* Validate kernel version */
2020         ret = kernel_validate_version(&the_kernel_tracer_version, &the_kernel_tracer_abi_version);
2021         if (ret < 0) {
2022                 kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
2023                         LTTNG_KERNEL_TRACER_STATUS_ERR_VERSION_MISMATCH);
2024                 goto error_version;
2025         }
2026
2027         ret = modprobe_lttng_data();
2028         if (ret < 0) {
2029                 set_kernel_tracer_status_from_modules_ret(-ret);
2030                 goto error_modules;
2031         }
2032
2033         ret = kernel_supports_ring_buffer_snapshot_sample_positions();
2034         if (ret < 0) {
2035                 goto error_modules;
2036         }
2037         if (ret < 1) {
2038                 WARN("Kernel tracer does not support buffer monitoring. "
2039                      "The monitoring timer of channels in the kernel domain "
2040                      "will be set to 0 (disabled).");
2041         }
2042
2043         ret = kernel_supports_event_notifiers();
2044         if (ret < 0) {
2045                 ERR("Failed to check for kernel tracer event notifier support");
2046                 kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
2047                         LTTNG_KERNEL_TRACER_STATUS_ERR_NOTIFIER);
2048                 goto error_modules;
2049         }
2050         ret = kernel_create_event_notifier_group(&kernel_tracer_event_notifier_group_fd);
2051         if (ret < 0) {
2052                 /* This is not fatal. */
2053                 WARN("Failed to create kernel event notifier group");
2054                 kernel_tracer_event_notifier_group_fd = -1;
2055         } else {
2056                 enum event_notifier_error_accounting_status error_accounting_status;
2057                 enum lttng_error_code error_code_ret =
2058                         kernel_create_event_notifier_group_notification_fd(
2059                                 &kernel_tracer_event_notifier_group_notification_fd);
2060
2061                 if (error_code_ret != LTTNG_OK) {
2062                         kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
2063                                 LTTNG_KERNEL_TRACER_STATUS_ERR_NOTIFIER);
2064                         goto error_modules;
2065                 }
2066
2067                 error_accounting_status = event_notifier_error_accounting_register_kernel(
2068                         kernel_tracer_event_notifier_group_fd);
2069                 if (error_accounting_status != EVENT_NOTIFIER_ERROR_ACCOUNTING_STATUS_OK) {
2070                         ERR("Failed to initialize event notifier error accounting for kernel tracer");
2071                         error_code_ret = LTTNG_ERR_EVENT_NOTIFIER_ERROR_ACCOUNTING;
2072                         kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
2073                                 LTTNG_KERNEL_TRACER_STATUS_ERR_NOTIFIER);
2074                         goto error_modules;
2075                 }
2076
2077                 kernel_token_to_event_notifier_rule_ht = cds_lfht_new(
2078                         DEFAULT_HT_SIZE, 1, 0, CDS_LFHT_AUTO_RESIZE | CDS_LFHT_ACCOUNTING, nullptr);
2079                 if (!kernel_token_to_event_notifier_rule_ht) {
2080                         kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
2081                                 LTTNG_KERNEL_TRACER_STATUS_ERR_NOTIFIER);
2082                         goto error_token_ht;
2083                 }
2084         }
2085
2086         DBG("Kernel tracer initialized: kernel tracer fd = %d, event notifier group fd = %d, event notifier group notification fd = %d",
2087             kernel_tracer_fd,
2088             kernel_tracer_event_notifier_group_fd,
2089             kernel_tracer_event_notifier_group_notification_fd);
2090
2091         ret = syscall_init_table(kernel_tracer_fd);
2092         if (ret < 0) {
2093                 ERR("Unable to populate syscall table. Syscall tracing won't "
2094                     "work for this session daemon.");
2095         }
2096
2097         kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(LTTNG_KERNEL_TRACER_STATUS_INITIALIZED);
2098         return 0;
2099
2100 error_version:
2101         modprobe_remove_lttng_control();
2102         ret = close(kernel_tracer_fd);
2103         if (ret) {
2104                 PERROR("Failed to close kernel tracer file descriptor: fd = %d", kernel_tracer_fd);
2105         }
2106
2107         kernel_tracer_fd = -1;
2108         return LTTNG_ERR_KERN_VERSION;
2109
2110 error_token_ht:
2111         ret = close(kernel_tracer_event_notifier_group_notification_fd);
2112         if (ret) {
2113                 PERROR("Failed to close kernel tracer event notifier group notification file descriptor: fd = %d",
2114                        kernel_tracer_event_notifier_group_notification_fd);
2115         }
2116
2117         kernel_tracer_event_notifier_group_notification_fd = -1;
2118
2119 error_modules:
2120         ret = close(kernel_tracer_event_notifier_group_fd);
2121         if (ret) {
2122                 PERROR("Failed to close kernel tracer event notifier group file descriptor: fd = %d",
2123                        kernel_tracer_event_notifier_group_fd);
2124         }
2125
2126         kernel_tracer_event_notifier_group_fd = -1;
2127
2128         ret = close(kernel_tracer_fd);
2129         if (ret) {
2130                 PERROR("Failed to close kernel tracer file descriptor: fd = %d", kernel_tracer_fd);
2131         }
2132
2133         kernel_tracer_fd = -1;
2134
2135 error_open:
2136         modprobe_remove_lttng_control();
2137
2138 error:
2139         WARN("No kernel tracer available");
2140         kernel_tracer_fd = -1;
2141         if (!is_root) {
2142                 kernel_tracer_status = nonstd::optional<enum lttng_kernel_tracer_status>(
2143                         LTTNG_KERNEL_TRACER_STATUS_ERR_NEED_ROOT);
2144                 return LTTNG_ERR_NEED_ROOT_SESSIOND;
2145         } else {
2146                 return LTTNG_ERR_KERN_NA;
2147         }
2148 }
2149
2150 void cleanup_kernel_tracer()
2151 {
2152         DBG2("Closing kernel event notifier group notification file descriptor");
2153         if (kernel_tracer_event_notifier_group_notification_fd >= 0) {
2154                 int ret = notification_thread_command_remove_tracer_event_source(
2155                         the_notification_thread_handle,
2156                         kernel_tracer_event_notifier_group_notification_fd);
2157                 if (ret != LTTNG_OK) {
2158                         ERR("Failed to remove kernel event notifier notification from notification thread");
2159                 }
2160
2161                 ret = close(kernel_tracer_event_notifier_group_notification_fd);
2162                 if (ret) {
2163                         PERROR("Failed to close kernel event notifier group notification file descriptor: fd = %d",
2164                                kernel_tracer_event_notifier_group_notification_fd);
2165                 }
2166
2167                 kernel_tracer_event_notifier_group_notification_fd = -1;
2168         }
2169
2170         if (kernel_token_to_event_notifier_rule_ht) {
2171                 const int ret = cds_lfht_destroy(kernel_token_to_event_notifier_rule_ht, nullptr);
2172                 LTTNG_ASSERT(ret == 0);
2173         }
2174
2175         DBG2("Closing kernel event notifier group file descriptor");
2176         if (kernel_tracer_event_notifier_group_fd >= 0) {
2177                 const int ret = close(kernel_tracer_event_notifier_group_fd);
2178
2179                 if (ret) {
2180                         PERROR("Failed to close kernel event notifier group file descriptor: fd = %d",
2181                                kernel_tracer_event_notifier_group_fd);
2182                 }
2183
2184                 kernel_tracer_event_notifier_group_fd = -1;
2185         }
2186
2187         DBG2("Closing kernel fd");
2188         if (kernel_tracer_fd >= 0) {
2189                 const int ret = close(kernel_tracer_fd);
2190
2191                 if (ret) {
2192                         PERROR("Failed to close kernel tracer file descriptor: fd = %d",
2193                                kernel_tracer_fd);
2194                 }
2195
2196                 kernel_tracer_fd = -1;
2197         }
2198
2199         kernel_tracer_status = nonstd::nullopt;
2200         free(syscall_table);
2201 }
2202
2203 bool kernel_tracer_is_initialized()
2204 {
2205         return kernel_tracer_fd >= 0;
2206 }
2207
2208 /*
2209  *  Clear a kernel session.
2210  *
2211  * Return LTTNG_OK on success or else an LTTng error code.
2212  */
2213 enum lttng_error_code kernel_clear_session(struct ltt_session *session)
2214 {
2215         int ret;
2216         enum lttng_error_code status = LTTNG_OK;
2217         struct consumer_socket *socket;
2218         struct lttng_ht_iter iter;
2219         struct ltt_kernel_session *ksess = session->kernel_session;
2220
2221         LTTNG_ASSERT(ksess);
2222         LTTNG_ASSERT(ksess->consumer);
2223
2224         DBG("Clear kernel session %s (session %" PRIu64 ")", session->name, session->id);
2225
2226         if (ksess->active) {
2227                 ERR("Expecting inactive session %s (%" PRIu64 ")", session->name, session->id);
2228                 status = LTTNG_ERR_FATAL;
2229                 goto end;
2230         }
2231
2232         {
2233                 /*
2234                  * Note that this loop will end after one iteration given that there is
2235                  * only one kernel consumer.
2236                  */
2237                 lttng::urcu::read_lock_guard read_lock;
2238
2239                 cds_lfht_for_each_entry (
2240                         ksess->consumer->socks->ht, &iter.iter, socket, node.node) {
2241                         struct ltt_kernel_channel *chan;
2242
2243                         /* For each channel, ask the consumer to clear it. */
2244                         cds_list_for_each_entry (chan, &ksess->channel_list.head, list) {
2245                                 DBG("Clear kernel channel %" PRIu64 ", session %s",
2246                                     chan->key,
2247                                     session->name);
2248                                 ret = consumer_clear_channel(socket, chan->key);
2249                                 if (ret < 0) {
2250                                         goto error;
2251                                 }
2252                         }
2253
2254                         if (!ksess->metadata) {
2255                                 /*
2256                                  * Nothing to do for the metadata.
2257                                  * This is a snapshot session.
2258                                  * The metadata is genererated on the fly.
2259                                  */
2260                                 continue;
2261                         }
2262
2263                         /*
2264                          * Clear the metadata channel.
2265                          * Metadata channel is not cleared per se but we still need to
2266                          * perform a rotation operation on it behind the scene.
2267                          */
2268                         ret = consumer_clear_channel(socket, ksess->metadata->key);
2269                         if (ret < 0) {
2270                                 goto error;
2271                         }
2272                 }
2273         }
2274
2275         goto end;
2276 error:
2277         switch (-ret) {
2278         case LTTCOMM_CONSUMERD_RELAYD_CLEAR_DISALLOWED:
2279                 status = LTTNG_ERR_CLEAR_RELAY_DISALLOWED;
2280                 break;
2281         default:
2282                 status = LTTNG_ERR_CLEAR_FAIL_CONSUMER;
2283                 break;
2284         }
2285 end:
2286         return status;
2287 }
2288
2289 enum lttng_error_code
2290 kernel_create_event_notifier_group_notification_fd(int *event_notifier_group_notification_fd)
2291 {
2292         int local_fd = -1, ret;
2293         enum lttng_error_code error_code_ret;
2294
2295         LTTNG_ASSERT(event_notifier_group_notification_fd);
2296
2297         ret = kernctl_create_event_notifier_group_notification_fd(
2298                 kernel_tracer_event_notifier_group_fd);
2299         if (ret < 0) {
2300                 PERROR("Failed to create kernel event notifier group notification file descriptor");
2301                 error_code_ret = LTTNG_ERR_EVENT_NOTIFIER_GROUP_NOTIFICATION_FD;
2302                 goto error;
2303         }
2304
2305         local_fd = ret;
2306
2307         /* Prevent fd duplication after execlp(). */
2308         ret = fcntl(local_fd, F_SETFD, FD_CLOEXEC);
2309         if (ret < 0) {
2310                 PERROR("Failed to set FD_CLOEXEC on kernel event notifier group notification file descriptor: fd = %d",
2311                        local_fd);
2312                 error_code_ret = LTTNG_ERR_EVENT_NOTIFIER_GROUP_NOTIFICATION_FD;
2313                 goto error;
2314         }
2315
2316         DBG("Created kernel notifier group notification file descriptor: fd = %d", local_fd);
2317         error_code_ret = LTTNG_OK;
2318         *event_notifier_group_notification_fd = local_fd;
2319         local_fd = -1;
2320
2321 error:
2322         if (local_fd >= 0) {
2323                 ret = close(local_fd);
2324                 if (ret) {
2325                         PERROR("Failed to close kernel event notifier group notification file descriptor: fd = %d",
2326                                local_fd);
2327                 }
2328         }
2329
2330         return error_code_ret;
2331 }
2332
2333 enum lttng_error_code
2334 kernel_destroy_event_notifier_group_notification_fd(int event_notifier_group_notification_fd)
2335 {
2336         enum lttng_error_code ret_code = LTTNG_OK;
2337
2338         DBG("Closing event notifier group notification file descriptor: fd = %d",
2339             event_notifier_group_notification_fd);
2340         if (event_notifier_group_notification_fd >= 0) {
2341                 const int ret = close(event_notifier_group_notification_fd);
2342                 if (ret) {
2343                         PERROR("Failed to close event notifier group notification file descriptor: fd = %d",
2344                                event_notifier_group_notification_fd);
2345                 }
2346         }
2347
2348         return ret_code;
2349 }
2350
2351 static unsigned long hash_trigger(const struct lttng_trigger *trigger)
2352 {
2353         const struct lttng_condition *condition = lttng_trigger_get_const_condition(trigger);
2354
2355         return lttng_condition_hash(condition);
2356 }
2357
2358 static int match_trigger(struct cds_lfht_node *node, const void *key)
2359 {
2360         const struct ltt_kernel_event_notifier_rule *event_notifier_rule;
2361         const struct lttng_trigger *trigger = (lttng_trigger *) key;
2362
2363         event_notifier_rule =
2364                 caa_container_of(node, const struct ltt_kernel_event_notifier_rule, ht_node);
2365
2366         return lttng_trigger_is_equal(trigger, event_notifier_rule->trigger);
2367 }
2368
2369 static enum lttng_error_code kernel_create_event_notifier_rule(
2370         struct lttng_trigger *trigger, const struct lttng_credentials *creds, uint64_t token)
2371 {
2372         int err, fd, ret = 0;
2373         enum lttng_error_code error_code_ret;
2374         enum lttng_condition_status condition_status;
2375         enum lttng_condition_type condition_type;
2376         enum lttng_event_rule_type event_rule_type;
2377         struct ltt_kernel_event_notifier_rule *event_notifier_rule;
2378         struct lttng_kernel_abi_event_notifier kernel_event_notifier = {};
2379         unsigned int capture_bytecode_count = 0, i;
2380         const struct lttng_condition *condition = nullptr;
2381         const struct lttng_event_rule *event_rule = nullptr;
2382         enum lttng_condition_status cond_status;
2383
2384         LTTNG_ASSERT(trigger);
2385
2386         condition = lttng_trigger_get_const_condition(trigger);
2387         LTTNG_ASSERT(condition);
2388
2389         condition_type = lttng_condition_get_type(condition);
2390         LTTNG_ASSERT(condition_type == LTTNG_CONDITION_TYPE_EVENT_RULE_MATCHES);
2391
2392         /* Does not acquire a reference. */
2393         condition_status = lttng_condition_event_rule_matches_get_rule(condition, &event_rule);
2394         LTTNG_ASSERT(condition_status == LTTNG_CONDITION_STATUS_OK);
2395         LTTNG_ASSERT(event_rule);
2396
2397         event_rule_type = lttng_event_rule_get_type(event_rule);
2398         LTTNG_ASSERT(event_rule_type != LTTNG_EVENT_RULE_TYPE_UNKNOWN);
2399
2400         error_code_ret = trace_kernel_create_event_notifier_rule(
2401                 trigger,
2402                 token,
2403                 lttng_condition_event_rule_matches_get_error_counter_index(condition),
2404                 &event_notifier_rule);
2405         if (error_code_ret != LTTNG_OK) {
2406                 goto error;
2407         }
2408
2409         error_code_ret = trace_kernel_init_event_notifier_from_event_rule(event_rule,
2410                                                                           &kernel_event_notifier);
2411         if (error_code_ret != LTTNG_OK) {
2412                 goto free_event;
2413         }
2414
2415         kernel_event_notifier.event.token = event_notifier_rule->token;
2416         kernel_event_notifier.error_counter_idx =
2417                 lttng_condition_event_rule_matches_get_error_counter_index(condition);
2418
2419         fd = kernctl_create_event_notifier(kernel_tracer_event_notifier_group_fd,
2420                                            &kernel_event_notifier);
2421         if (fd < 0) {
2422                 switch (-fd) {
2423                 case EEXIST:
2424                         error_code_ret = LTTNG_ERR_KERN_EVENT_EXIST;
2425                         break;
2426                 case ENOSYS:
2427                         WARN("Failed to create kernel event notifier: not notifier type not implemented");
2428                         error_code_ret = LTTNG_ERR_KERN_EVENT_ENOSYS;
2429                         break;
2430                 case ENOENT:
2431                         WARN("Failed to create kernel event notifier: not found: name = '%s'",
2432                              kernel_event_notifier.event.name);
2433                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2434                         break;
2435                 default:
2436                         PERROR("Failed to create kernel event notifier: error code = %d, name = '%s'",
2437                                fd,
2438                                kernel_event_notifier.event.name);
2439                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2440                 }
2441                 goto free_event;
2442         }
2443
2444         event_notifier_rule->fd = fd;
2445         /* Prevent fd duplication after execlp(). */
2446         err = fcntl(event_notifier_rule->fd, F_SETFD, FD_CLOEXEC);
2447         if (err < 0) {
2448                 PERROR("Failed to set FD_CLOEXEC on kernel event notifier file descriptor: fd = %d",
2449                        fd);
2450                 error_code_ret = LTTNG_ERR_FATAL;
2451                 goto set_cloexec_error;
2452         }
2453
2454         if (event_notifier_rule->filter) {
2455                 err = kernctl_filter(event_notifier_rule->fd, event_notifier_rule->filter);
2456                 if (err < 0) {
2457                         switch (-err) {
2458                         case ENOMEM:
2459                                 error_code_ret = LTTNG_ERR_FILTER_NOMEM;
2460                                 break;
2461                         default:
2462                                 error_code_ret = LTTNG_ERR_FILTER_INVAL;
2463                                 break;
2464                         }
2465                         goto filter_error;
2466                 }
2467         }
2468
2469         if (lttng_event_rule_get_type(event_rule) == LTTNG_EVENT_RULE_TYPE_KERNEL_UPROBE) {
2470                 ret = userspace_probe_event_rule_add_callsites(
2471                         event_rule, creds, event_notifier_rule->fd);
2472                 if (ret) {
2473                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2474                         goto add_callsite_error;
2475                 }
2476         }
2477
2478         /* Set the capture bytecode if any. */
2479         cond_status = lttng_condition_event_rule_matches_get_capture_descriptor_count(
2480                 condition, &capture_bytecode_count);
2481         LTTNG_ASSERT(cond_status == LTTNG_CONDITION_STATUS_OK);
2482
2483         for (i = 0; i < capture_bytecode_count; i++) {
2484                 const struct lttng_bytecode *capture_bytecode =
2485                         lttng_condition_event_rule_matches_get_capture_bytecode_at_index(condition,
2486                                                                                          i);
2487
2488                 if (capture_bytecode == nullptr) {
2489                         ERR("Unexpected NULL capture bytecode on condition");
2490                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2491                         goto capture_error;
2492                 }
2493
2494                 ret = kernctl_capture(event_notifier_rule->fd, capture_bytecode);
2495                 if (ret < 0) {
2496                         ERR("Failed to set capture bytecode on event notifier rule fd: fd = %d",
2497                             event_notifier_rule->fd);
2498                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2499                         goto capture_error;
2500                 }
2501         }
2502
2503         err = kernctl_enable(event_notifier_rule->fd);
2504         if (err < 0) {
2505                 switch (-err) {
2506                 case EEXIST:
2507                         error_code_ret = LTTNG_ERR_KERN_EVENT_EXIST;
2508                         break;
2509                 default:
2510                         PERROR("enable kernel event notifier");
2511                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2512                         break;
2513                 }
2514                 goto enable_error;
2515         }
2516
2517         /* Add trigger to kernel token mapping in the hash table. */
2518         {
2519                 lttng::urcu::read_lock_guard read_lock;
2520                 cds_lfht_add(kernel_token_to_event_notifier_rule_ht,
2521                              hash_trigger(trigger),
2522                              &event_notifier_rule->ht_node);
2523         }
2524
2525         DBG("Created kernel event notifier: name = '%s', fd = %d",
2526             kernel_event_notifier.event.name,
2527             event_notifier_rule->fd);
2528
2529         return LTTNG_OK;
2530
2531 capture_error:
2532 add_callsite_error:
2533 enable_error:
2534 set_cloexec_error:
2535 filter_error:
2536 {
2537         const int close_ret = close(event_notifier_rule->fd);
2538
2539         if (close_ret) {
2540                 PERROR("Failed to close kernel event notifier file descriptor: fd = %d",
2541                        event_notifier_rule->fd);
2542         }
2543 }
2544 free_event:
2545         free(event_notifier_rule);
2546 error:
2547         return error_code_ret;
2548 }
2549
2550 enum lttng_error_code kernel_register_event_notifier(struct lttng_trigger *trigger,
2551                                                      const struct lttng_credentials *cmd_creds)
2552 {
2553         enum lttng_error_code ret;
2554         enum lttng_condition_status status;
2555         enum lttng_domain_type domain_type;
2556         const struct lttng_event_rule *event_rule;
2557         const struct lttng_condition *const condition = lttng_trigger_get_const_condition(trigger);
2558         const uint64_t token = lttng_trigger_get_tracer_token(trigger);
2559
2560         LTTNG_ASSERT(condition);
2561
2562         /* Does not acquire a reference to the event rule. */
2563         status = lttng_condition_event_rule_matches_get_rule(condition, &event_rule);
2564         LTTNG_ASSERT(status == LTTNG_CONDITION_STATUS_OK);
2565
2566         domain_type = lttng_event_rule_get_domain_type(event_rule);
2567         LTTNG_ASSERT(domain_type == LTTNG_DOMAIN_KERNEL);
2568
2569         ret = kernel_create_event_notifier_rule(trigger, cmd_creds, token);
2570         if (ret != LTTNG_OK) {
2571                 ERR("Failed to create kernel event notifier rule");
2572         }
2573
2574         return ret;
2575 }
2576
2577 enum lttng_error_code kernel_unregister_event_notifier(const struct lttng_trigger *trigger)
2578 {
2579         struct ltt_kernel_event_notifier_rule *token_event_rule_element;
2580         struct cds_lfht_node *node;
2581         struct cds_lfht_iter iter;
2582         enum lttng_error_code error_code_ret;
2583         int ret;
2584
2585         lttng::urcu::read_lock_guard read_lock;
2586
2587         cds_lfht_lookup(kernel_token_to_event_notifier_rule_ht,
2588                         hash_trigger(trigger),
2589                         match_trigger,
2590                         trigger,
2591                         &iter);
2592
2593         node = cds_lfht_iter_get_node(&iter);
2594         if (!node) {
2595                 error_code_ret = LTTNG_ERR_TRIGGER_NOT_FOUND;
2596                 goto error;
2597         }
2598
2599         token_event_rule_element =
2600                 caa_container_of(node, struct ltt_kernel_event_notifier_rule, ht_node);
2601
2602         ret = kernel_disable_event_notifier_rule(token_event_rule_element);
2603         if (ret) {
2604                 error_code_ret = LTTNG_ERR_FATAL;
2605                 goto error;
2606         }
2607
2608         trace_kernel_destroy_event_notifier_rule(token_event_rule_element);
2609         error_code_ret = LTTNG_OK;
2610
2611 error:
2612
2613         return error_code_ret;
2614 }
2615
2616 int kernel_get_notification_fd()
2617 {
2618         return kernel_tracer_event_notifier_group_notification_fd;
2619 }