src/bin/lttng-sessiond/kernel.cpp

   1 /*
   2  * Copyright (C) 2011 EfficiOS Inc.
   3  *
   4  * SPDX-License-Identifier: GPL-2.0-only
   5  *
   6  */
   7
   8 #define _LGPL_SOURCE
   9 #include "condition-internal.hpp"
  10 #include "consumer.hpp"
  11 #include "event-notifier-error-accounting.hpp"
  12 #include "kern-modules.hpp"
  13 #include "kernel-consumer.hpp"
  14 #include "kernel.hpp"
  15 #include "lttng-sessiond.hpp"
  16 #include "lttng-syscall.hpp"
  17 #include "modprobe.hpp"
  18 #include "notification-thread-commands.hpp"
  19 #include "sessiond-config.hpp"
  20 #include "tracker.hpp"
  21 #include "utils.hpp"
  22
  23 #include <common/common.hpp>
  24 #include <common/hashtable/utils.hpp>
  25 #include <common/kernel-ctl/kernel-ctl.hpp>
  26 #include <common/kernel-ctl/kernel-ioctl.hpp>
  27 #include <common/sessiond-comm/sessiond-comm.hpp>
  28 #include <common/trace-chunk.hpp>
  29 #include <common/tracker.hpp>
  30 #include <common/urcu.hpp>
  31 #include <common/utils.hpp>
  32
  33 #include <lttng/condition/event-rule-matches-internal.hpp>
  34 #include <lttng/condition/event-rule-matches.h>
  35 #include <lttng/event-rule/event-rule-internal.hpp>
  36 #include <lttng/event-rule/event-rule.h>
  37 #include <lttng/event-rule/kernel-uprobe-internal.hpp>
  38 #include <lttng/event.h>
  39 #include <lttng/lttng-error.h>
  40 #include <lttng/tracker.h>
  41 #include <lttng/userspace-probe-internal.hpp>
  42 #include <lttng/userspace-probe.h>
  43
  44 #include <fcntl.h>
  45 #include <inttypes.h>
  46 #include <stdio.h>
  47 #include <stdlib.h>
  48 #include <string.h>
  49 #include <sys/types.h>
  50 #include <unistd.h>
  51
  52 /*
  53  * Key used to reference a channel between the sessiond and the consumer. This
  54  * is only read and updated with the session_list lock held.
  55  */
  56 static uint64_t next_kernel_channel_key;
  57
  58 static const char *module_proc_lttng = "/proc/lttng";
  59
  60 static int kernel_tracer_fd = -1;
  61 static int kernel_tracer_event_notifier_group_fd = -1;
  62 static int kernel_tracer_event_notifier_group_notification_fd = -1;
  63 static struct cds_lfht *kernel_token_to_event_notifier_rule_ht;
  64
  65 namespace {
  66 /*
  67  * On some architectures, calling convention details are embedded in the symbol
  68  * addresses. Uprobe requires a "clean" symbol offset (or at least, an address
  69  * where an instruction boundary would be legal) to add
  70  * instrumentation. sanitize_uprobe_offset implements that sanitization logic on
  71  * a per-architecture basis.
  72  */
  73 #if defined(__arm__) || defined(__aarch64__)
  74 static inline uint64_t sanitize_uprobe_offset(uint64_t raw_offset)
  75 {
  76         /*
  77          * The least significant bit is used when branching to switch to thumb
  78          * ISA. However, it's an invalid address for us; mask the least
  79          * significant bit.
  80          */
  81         return raw_offset &= ~0b1;
  82 }
  83 #else /* defined(__arm__) || defined(__aarch64__) */
  84 static inline uint64_t sanitize_uprobe_offset(uint64_t raw_offset)
  85 {
  86         return raw_offset;
  87 }
  88 #endif
  89 } /* namespace */
  90
  91 /*
  92  * Add context on a kernel channel.
  93  *
  94  * Assumes the ownership of ctx.
  95  */
  96 int kernel_add_channel_context(struct ltt_kernel_channel *chan, struct ltt_kernel_context *ctx)
  97 {
  98         int ret;
  99
 100         LTTNG_ASSERT(chan);
 101         LTTNG_ASSERT(ctx);
 102
 103         DBG("Adding context to channel %s", chan->channel->name);
 104         ret = kernctl_add_context(chan->fd, &ctx->ctx);
 105         if (ret < 0) {
 106                 switch (-ret) {
 107                 case ENOSYS:
 108                         /* Exists but not available for this kernel */
 109                         ret = LTTNG_ERR_KERN_CONTEXT_UNAVAILABLE;
 110                         goto error;
 111                 case EEXIST:
 112                         /* If EEXIST, we just ignore the error */
 113                         ret = 0;
 114                         goto end;
 115                 default:
 116                         PERROR("add context ioctl");
 117                         ret = LTTNG_ERR_KERN_CONTEXT_FAIL;
 118                         goto error;
 119                 }
 120         }
 121         ret = 0;
 122
 123 end:
 124         cds_list_add_tail(&ctx->list, &chan->ctx_list);
 125         ctx->in_list = true;
 126         ctx = nullptr;
 127 error:
 128         if (ctx) {
 129                 trace_kernel_destroy_context(ctx);
 130         }
 131         return ret;
 132 }
 133
 134 /*
 135  * Create a new kernel session, register it to the kernel tracer and add it to
 136  * the session daemon session.
 137  */
 138 int kernel_create_session(struct ltt_session *session)
 139 {
 140         int ret;
 141         struct ltt_kernel_session *lks;
 142
 143         LTTNG_ASSERT(session);
 144
 145         /* Allocate data structure */
 146         lks = trace_kernel_create_session();
 147         if (lks == nullptr) {
 148                 ret = -1;
 149                 goto error;
 150         }
 151
 152         /* Kernel tracer session creation */
 153         ret = kernctl_create_session(kernel_tracer_fd);
 154         if (ret < 0) {
 155                 PERROR("ioctl kernel create session");
 156                 goto error;
 157         }
 158
 159         lks->fd = ret;
 160         /* Prevent fd duplication after execlp() */
 161         ret = fcntl(lks->fd, F_SETFD, FD_CLOEXEC);
 162         if (ret < 0) {
 163                 PERROR("fcntl session fd");
 164         }
 165
 166         lks->id = session->id;
 167         lks->consumer_fds_sent = 0;
 168         session->kernel_session = lks;
 169
 170         DBG("Kernel session created (fd: %d)", lks->fd);
 171
 172         /*
 173          * This is necessary since the creation time is present in the session
 174          * name when it is generated.
 175          */
 176         if (session->has_auto_generated_name) {
 177                 ret = kernctl_session_set_name(lks->fd, DEFAULT_SESSION_NAME);
 178         } else {
 179                 ret = kernctl_session_set_name(lks->fd, session->name);
 180         }
 181         if (ret) {
 182                 WARN("Could not set kernel session name for session %" PRIu64 " name: %s",
 183                      session->id,
 184                      session->name);
 185         }
 186
 187         ret = kernctl_session_set_creation_time(lks->fd, session->creation_time);
 188         if (ret) {
 189                 WARN("Could not set kernel session creation time for session %" PRIu64 " name: %s",
 190                      session->id,
 191                      session->name);
 192         }
 193
 194         return 0;
 195
 196 error:
 197         if (lks) {
 198                 trace_kernel_destroy_session(lks);
 199                 trace_kernel_free_session(lks);
 200         }
 201         return ret;
 202 }
 203
 204 /*
 205  * Create a kernel channel, register it to the kernel tracer and add it to the
 206  * kernel session.
 207  */
 208 int kernel_create_channel(struct ltt_kernel_session *session, struct lttng_channel *chan)
 209 {
 210         int ret;
 211         struct ltt_kernel_channel *lkc;
 212
 213         LTTNG_ASSERT(session);
 214         LTTNG_ASSERT(chan);
 215
 216         /* Allocate kernel channel */
 217         lkc = trace_kernel_create_channel(chan);
 218         if (lkc == nullptr) {
 219                 goto error;
 220         }
 221
 222         DBG3("Kernel create channel %s with attr: %d, %" PRIu64 ", %" PRIu64 ", %u, %u, %d, %d",
 223              chan->name,
 224              lkc->channel->attr.overwrite,
 225              lkc->channel->attr.subbuf_size,
 226              lkc->channel->attr.num_subbuf,
 227              lkc->channel->attr.switch_timer_interval,
 228              lkc->channel->attr.read_timer_interval,
 229              lkc->channel->attr.live_timer_interval,
 230              lkc->channel->attr.output);
 231
 232         /* Kernel tracer channel creation */
 233         ret = kernctl_create_channel(session->fd, &lkc->channel->attr);
 234         if (ret < 0) {
 235                 PERROR("ioctl kernel create channel");
 236                 goto error;
 237         }
 238
 239         /* Setup the channel fd */
 240         lkc->fd = ret;
 241         /* Prevent fd duplication after execlp() */
 242         ret = fcntl(lkc->fd, F_SETFD, FD_CLOEXEC);
 243         if (ret < 0) {
 244                 PERROR("fcntl session fd");
 245         }
 246
 247         /* Add channel to session */
 248         cds_list_add(&lkc->list, &session->channel_list.head);
 249         session->channel_count++;
 250         lkc->session = session;
 251         lkc->key = ++next_kernel_channel_key;
 252
 253         DBG("Kernel channel %s created (fd: %d, key: %" PRIu64 ")",
 254             lkc->channel->name,
 255             lkc->fd,
 256             lkc->key);
 257
 258         return 0;
 259
 260 error:
 261         if (lkc) {
 262                 free(lkc->channel);
 263                 free(lkc);
 264         }
 265         return -1;
 266 }
 267
 268 /*
 269  * Create a kernel event notifier group, register it to the kernel tracer and
 270  * add it to the kernel session.
 271  */
 272 static int kernel_create_event_notifier_group(int *event_notifier_group_fd)
 273 {
 274         int ret;
 275         int local_fd = -1;
 276
 277         LTTNG_ASSERT(event_notifier_group_fd);
 278
 279         /* Kernel event notifier group creation. */
 280         ret = kernctl_create_event_notifier_group(kernel_tracer_fd);
 281         if (ret < 0) {
 282                 PERROR("Failed to create kernel event notifier group");
 283                 ret = -1;
 284                 goto error;
 285         }
 286
 287         local_fd = ret;
 288
 289         /* Prevent fd duplication after execlp(). */
 290         ret = fcntl(local_fd, F_SETFD, FD_CLOEXEC);
 291         if (ret < 0) {
 292                 PERROR("Failed to set FD_CLOEXEC on kernel event notifier group file descriptor: fd = %d",
 293                        local_fd);
 294                 goto error;
 295         }
 296
 297         DBG("Created kernel event notifier group: fd = %d", local_fd);
 298         *event_notifier_group_fd = local_fd;
 299         local_fd = -1;
 300         ret = 0;
 301 error:
 302         if (local_fd >= 0) {
 303                 ret = close(local_fd);
 304                 if (ret) {
 305                         PERROR("Failed to close kernel event notifier group file descriptor: fd = %d",
 306                                local_fd);
 307                 }
 308         }
 309
 310         return ret;
 311 }
 312
 313 /*
 314  * Compute the offset of the instrumentation byte in the binary based on the
 315  * function probe location using the ELF lookup method.
 316  *
 317  * Returns 0 on success and set the offset out parameter to the offset of the
 318  * elf symbol
 319  * Returns -1 on error
 320  */
 321 static int extract_userspace_probe_offset_function_elf(
 322         const struct lttng_userspace_probe_location *probe_location,
 323         uid_t uid,
 324         gid_t gid,
 325         uint64_t *offset)
 326 {
 327         int fd;
 328         int ret = 0;
 329         const char *symbol = nullptr;
 330         const struct lttng_userspace_probe_location_lookup_method *lookup = nullptr;
 331         enum lttng_userspace_probe_location_lookup_method_type lookup_method_type;
 332
 333         LTTNG_ASSERT(lttng_userspace_probe_location_get_type(probe_location) ==
 334                      LTTNG_USERSPACE_PROBE_LOCATION_TYPE_FUNCTION);
 335
 336         lookup = lttng_userspace_probe_location_get_lookup_method(probe_location);
 337         if (!lookup) {
 338                 ret = -1;
 339                 goto end;
 340         }
 341
 342         lookup_method_type = lttng_userspace_probe_location_lookup_method_get_type(lookup);
 343
 344         LTTNG_ASSERT(lookup_method_type ==
 345                      LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_FUNCTION_ELF);
 346
 347         symbol = lttng_userspace_probe_location_function_get_function_name(probe_location);
 348         if (!symbol) {
 349                 ret = -1;
 350                 goto end;
 351         }
 352
 353         fd = lttng_userspace_probe_location_function_get_binary_fd(probe_location);
 354         if (fd < 0) {
 355                 ret = -1;
 356                 goto end;
 357         }
 358
 359         ret = run_as_extract_elf_symbol_offset(fd, symbol, uid, gid, offset);
 360         if (ret < 0) {
 361                 DBG("userspace probe offset calculation failed for "
 362                     "function %s",
 363                     symbol);
 364                 goto end;
 365         }
 366
 367         DBG("userspace probe elf offset for %s is 0x%jd", symbol, (intmax_t) (*offset));
 368 end:
 369         return ret;
 370 }
 371
 372 /*
 373  * Compute the offsets of the instrumentation bytes in the binary based on the
 374  * tracepoint probe location using the SDT lookup method. This function
 375  * allocates the offsets buffer, the caller must free it.
 376  *
 377  * Returns 0 on success and set the offset out parameter to the offsets of the
 378  * SDT tracepoint.
 379  * Returns -1 on error.
 380  */
 381 static int extract_userspace_probe_offset_tracepoint_sdt(
 382         const struct lttng_userspace_probe_location *probe_location,
 383         uid_t uid,
 384         gid_t gid,
 385         uint64_t **offsets,
 386         uint32_t *offsets_count)
 387 {
 388         enum lttng_userspace_probe_location_lookup_method_type lookup_method_type;
 389         const struct lttng_userspace_probe_location_lookup_method *lookup = nullptr;
 390         const char *probe_name = nullptr, *provider_name = nullptr;
 391         int ret = 0;
 392         int fd, i;
 393
 394         LTTNG_ASSERT(lttng_userspace_probe_location_get_type(probe_location) ==
 395                      LTTNG_USERSPACE_PROBE_LOCATION_TYPE_TRACEPOINT);
 396
 397         lookup = lttng_userspace_probe_location_get_lookup_method(probe_location);
 398         if (!lookup) {
 399                 ret = -1;
 400                 goto end;
 401         }
 402
 403         lookup_method_type = lttng_userspace_probe_location_lookup_method_get_type(lookup);
 404
 405         LTTNG_ASSERT(lookup_method_type ==
 406                      LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_TRACEPOINT_SDT);
 407
 408         probe_name = lttng_userspace_probe_location_tracepoint_get_probe_name(probe_location);
 409         if (!probe_name) {
 410                 ret = -1;
 411                 goto end;
 412         }
 413
 414         provider_name = lttng_userspace_probe_location_tracepoint_get_provider_name(probe_location);
 415         if (!provider_name) {
 416                 ret = -1;
 417                 goto end;
 418         }
 419
 420         fd = lttng_userspace_probe_location_tracepoint_get_binary_fd(probe_location);
 421         if (fd < 0) {
 422                 ret = -1;
 423                 goto end;
 424         }
 425
 426         ret = run_as_extract_sdt_probe_offsets(
 427                 fd, provider_name, probe_name, uid, gid, offsets, offsets_count);
 428         if (ret < 0) {
 429                 DBG("userspace probe offset calculation failed for sdt "
 430                     "probe %s:%s",
 431                     provider_name,
 432                     probe_name);
 433                 goto end;
 434         }
 435
 436         if (*offsets_count == 0) {
 437                 DBG("no userspace probe offset found");
 438                 goto end;
 439         }
 440
 441         DBG("%u userspace probe SDT offsets found for %s:%s at:",
 442             *offsets_count,
 443             provider_name,
 444             probe_name);
 445         for (i = 0; i < *offsets_count; i++) {
 446                 DBG("\t0x%jd", (intmax_t) ((*offsets)[i]));
 447         }
 448 end:
 449         return ret;
 450 }
 451
 452 static int userspace_probe_add_callsite(const struct lttng_userspace_probe_location *location,
 453                                         uid_t uid,
 454                                         gid_t gid,
 455                                         int fd)
 456 {
 457         const struct lttng_userspace_probe_location_lookup_method *lookup_method = nullptr;
 458         enum lttng_userspace_probe_location_lookup_method_type type;
 459         int ret;
 460
 461         lookup_method = lttng_userspace_probe_location_get_lookup_method(location);
 462         if (!lookup_method) {
 463                 ret = -1;
 464                 goto end;
 465         }
 466
 467         type = lttng_userspace_probe_location_lookup_method_get_type(lookup_method);
 468         switch (type) {
 469         case LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_FUNCTION_ELF:
 470         {
 471                 struct lttng_kernel_abi_event_callsite callsite;
 472                 uint64_t offset;
 473
 474                 ret = extract_userspace_probe_offset_function_elf(location, uid, gid, &offset);
 475                 if (ret) {
 476                         ret = LTTNG_ERR_PROBE_LOCATION_INVAL;
 477                         goto end;
 478                 }
 479
 480                 callsite.u.uprobe.offset = sanitize_uprobe_offset(offset);
 481                 ret = kernctl_add_callsite(fd, &callsite);
 482                 if (ret) {
 483                         WARN("Failed to add callsite to ELF userspace probe.");
 484                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 485                         goto end;
 486                 }
 487                 break;
 488         }
 489         case LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_TRACEPOINT_SDT:
 490         {
 491                 int i;
 492                 uint64_t *offsets = nullptr;
 493                 uint32_t offsets_count;
 494                 struct lttng_kernel_abi_event_callsite callsite;
 495
 496                 /*
 497                  * This call allocates the offsets buffer. This buffer must be freed
 498                  * by the caller
 499                  */
 500                 ret = extract_userspace_probe_offset_tracepoint_sdt(
 501                         location, uid, gid, &offsets, &offsets_count);
 502                 if (ret) {
 503                         ret = LTTNG_ERR_PROBE_LOCATION_INVAL;
 504                         goto end;
 505                 }
 506                 for (i = 0; i < offsets_count; i++) {
 507                         callsite.u.uprobe.offset = sanitize_uprobe_offset(offsets[i]);
 508                         ret = kernctl_add_callsite(fd, &callsite);
 509                         if (ret) {
 510                                 WARN("Failed to add callsite to SDT userspace probe");
 511                                 ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 512                                 free(offsets);
 513                                 goto end;
 514                         }
 515                 }
 516                 free(offsets);
 517                 break;
 518         }
 519         default:
 520                 ret = LTTNG_ERR_PROBE_LOCATION_INVAL;
 521                 goto end;
 522         }
 523 end:
 524         return ret;
 525 }
 526
 527 /*
 528  * Extract the offsets of the instrumentation point for the different lookup
 529  * methods.
 530  */
 531 static int userspace_probe_event_add_callsites(struct lttng_event *ev,
 532                                                struct ltt_kernel_session *session,
 533                                                int fd)
 534 {
 535         int ret;
 536         const struct lttng_userspace_probe_location *location = nullptr;
 537
 538         LTTNG_ASSERT(ev);
 539         LTTNG_ASSERT(ev->type == LTTNG_EVENT_USERSPACE_PROBE);
 540
 541         location = lttng_event_get_userspace_probe_location(ev);
 542         if (!location) {
 543                 ret = -1;
 544                 goto end;
 545         }
 546
 547         ret = userspace_probe_add_callsite(location, session->uid, session->gid, fd);
 548         if (ret) {
 549                 WARN("Failed to add callsite to userspace probe event '%s'", ev->name);
 550         }
 551
 552 end:
 553         return ret;
 554 }
 555
 556 /*
 557  * Extract the offsets of the instrumentation point for the different look-up
 558  * methods.
 559  */
 560 static int userspace_probe_event_rule_add_callsites(const struct lttng_event_rule *rule,
 561                                                     const struct lttng_credentials *creds,
 562                                                     int fd)
 563 {
 564         int ret;
 565         enum lttng_event_rule_status status;
 566         enum lttng_event_rule_type event_rule_type;
 567         const struct lttng_userspace_probe_location *location = nullptr;
 568
 569         LTTNG_ASSERT(rule);
 570         LTTNG_ASSERT(creds);
 571
 572         event_rule_type = lttng_event_rule_get_type(rule);
 573         LTTNG_ASSERT(event_rule_type == LTTNG_EVENT_RULE_TYPE_KERNEL_UPROBE);
 574
 575         status = lttng_event_rule_kernel_uprobe_get_location(rule, &location);
 576         if (status != LTTNG_EVENT_RULE_STATUS_OK || !location) {
 577                 ret = -1;
 578                 goto end;
 579         }
 580
 581         ret = userspace_probe_add_callsite(
 582                 location, lttng_credentials_get_uid(creds), lttng_credentials_get_gid(creds), fd);
 583         if (ret) {
 584                 WARN("Failed to add callsite to user space probe object: fd = %d", fd);
 585         }
 586
 587 end:
 588         return ret;
 589 }
 590
 591 /*
 592  * Create a kernel event, enable it to the kernel tracer and add it to the
 593  * channel event list of the kernel session.
 594  * We own filter_expression and filter.
 595  */
 596 int kernel_create_event(struct lttng_event *ev,
 597                         struct ltt_kernel_channel *channel,
 598                         char *filter_expression,
 599                         struct lttng_bytecode *filter)
 600 {
 601         int err, fd;
 602         enum lttng_error_code ret;
 603         struct ltt_kernel_event *event;
 604
 605         LTTNG_ASSERT(ev);
 606         LTTNG_ASSERT(channel);
 607
 608         /* We pass ownership of filter_expression and filter */
 609         ret = trace_kernel_create_event(ev, filter_expression, filter, &event);
 610         if (ret != LTTNG_OK) {
 611                 goto error;
 612         }
 613
 614         fd = kernctl_create_event(channel->fd, event->event);
 615         if (fd < 0) {
 616                 switch (-fd) {
 617                 case EEXIST:
 618                         ret = LTTNG_ERR_KERN_EVENT_EXIST;
 619                         break;
 620                 case ENOSYS:
 621                         WARN("Event type not implemented");
 622                         ret = LTTNG_ERR_KERN_EVENT_ENOSYS;
 623                         break;
 624                 case ENOENT:
 625                         WARN("Event %s not found!", ev->name);
 626                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 627                         break;
 628                 default:
 629                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 630                         PERROR("create event ioctl");
 631                 }
 632                 goto free_event;
 633         }
 634
 635         event->type = ev->type;
 636         event->fd = fd;
 637         /* Prevent fd duplication after execlp() */
 638         err = fcntl(event->fd, F_SETFD, FD_CLOEXEC);
 639         if (err < 0) {
 640                 PERROR("fcntl session fd");
 641         }
 642
 643         if (filter) {
 644                 err = kernctl_filter(event->fd, filter);
 645                 if (err < 0) {
 646                         switch (-err) {
 647                         case ENOMEM:
 648                                 ret = LTTNG_ERR_FILTER_NOMEM;
 649                                 break;
 650                         default:
 651                                 ret = LTTNG_ERR_FILTER_INVAL;
 652                                 break;
 653                         }
 654                         goto filter_error;
 655                 }
 656         }
 657
 658         if (ev->type == LTTNG_EVENT_USERSPACE_PROBE) {
 659                 ret = (lttng_error_code) userspace_probe_event_add_callsites(
 660                         ev, channel->session, event->fd);
 661                 if (ret) {
 662                         goto add_callsite_error;
 663                 }
 664         }
 665
 666         err = kernctl_enable(event->fd);
 667         if (err < 0) {
 668                 switch (-err) {
 669                 case EEXIST:
 670                         ret = LTTNG_ERR_KERN_EVENT_EXIST;
 671                         break;
 672                 default:
 673                         PERROR("enable kernel event");
 674                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 675                         break;
 676                 }
 677                 goto enable_error;
 678         }
 679
 680         /* Add event to event list */
 681         cds_list_add(&event->list, &channel->events_list.head);
 682         channel->event_count++;
 683
 684         DBG("Event %s created (fd: %d)", ev->name, event->fd);
 685
 686         return 0;
 687
 688 add_callsite_error:
 689 enable_error:
 690 filter_error:
 691 {
 692         int closeret;
 693
 694         closeret = close(event->fd);
 695         if (closeret) {
 696                 PERROR("close event fd");
 697         }
 698 }
 699 free_event:
 700         free(event);
 701 error:
 702         return ret;
 703 }
 704
 705 /*
 706  * Disable a kernel channel.
 707  */
 708 int kernel_disable_channel(struct ltt_kernel_channel *chan)
 709 {
 710         int ret;
 711
 712         LTTNG_ASSERT(chan);
 713
 714         ret = kernctl_disable(chan->fd);
 715         if (ret < 0) {
 716                 PERROR("disable chan ioctl");
 717                 goto error;
 718         }
 719
 720         chan->enabled = false;
 721         DBG("Kernel channel %s disabled (fd: %d, key: %" PRIu64 ")",
 722             chan->channel->name,
 723             chan->fd,
 724             chan->key);
 725
 726         return 0;
 727
 728 error:
 729         return ret;
 730 }
 731
 732 /*
 733  * Enable a kernel channel.
 734  */
 735 int kernel_enable_channel(struct ltt_kernel_channel *chan)
 736 {
 737         int ret;
 738
 739         LTTNG_ASSERT(chan);
 740
 741         ret = kernctl_enable(chan->fd);
 742         if (ret < 0 && ret != -EEXIST) {
 743                 PERROR("Enable kernel chan");
 744                 goto error;
 745         }
 746
 747         chan->enabled = true;
 748         DBG("Kernel channel %s enabled (fd: %d, key: %" PRIu64 ")",
 749             chan->channel->name,
 750             chan->fd,
 751             chan->key);
 752
 753         return 0;
 754
 755 error:
 756         return ret;
 757 }
 758
 759 /*
 760  * Enable a kernel event.
 761  */
 762 int kernel_enable_event(struct ltt_kernel_event *event)
 763 {
 764         int ret;
 765
 766         LTTNG_ASSERT(event);
 767
 768         ret = kernctl_enable(event->fd);
 769         if (ret < 0) {
 770                 switch (-ret) {
 771                 case EEXIST:
 772                         ret = LTTNG_ERR_KERN_EVENT_EXIST;
 773                         break;
 774                 default:
 775                         PERROR("enable kernel event");
 776                         break;
 777                 }
 778                 goto error;
 779         }
 780
 781         event->enabled = true;
 782         DBG("Kernel event %s enabled (fd: %d)", event->event->name, event->fd);
 783
 784         return 0;
 785
 786 error:
 787         return ret;
 788 }
 789
 790 /*
 791  * Disable a kernel event.
 792  */
 793 int kernel_disable_event(struct ltt_kernel_event *event)
 794 {
 795         int ret;
 796
 797         LTTNG_ASSERT(event);
 798
 799         ret = kernctl_disable(event->fd);
 800         if (ret < 0) {
 801                 PERROR("Failed to disable kernel event: name = '%s', fd = %d",
 802                        event->event->name,
 803                        event->fd);
 804                 goto error;
 805         }
 806
 807         event->enabled = false;
 808         DBG("Kernel event %s disabled (fd: %d)", event->event->name, event->fd);
 809
 810         return 0;
 811
 812 error:
 813         return ret;
 814 }
 815
 816 /*
 817  * Disable a kernel event notifier.
 818  */
 819 static int kernel_disable_event_notifier_rule(struct ltt_kernel_event_notifier_rule *event)
 820 {
 821         int ret;
 822
 823         LTTNG_ASSERT(event);
 824
 825         lttng::urcu::read_lock_guard read_lock;
 826         cds_lfht_del(kernel_token_to_event_notifier_rule_ht, &event->ht_node);
 827
 828         ret = kernctl_disable(event->fd);
 829         if (ret < 0) {
 830                 PERROR("Failed to disable kernel event notifier: fd = %d, token = %" PRIu64,
 831                        event->fd,
 832                        event->token);
 833                 goto error;
 834         }
 835
 836         event->enabled = false;
 837         DBG("Disabled kernel event notifier: fd = %d, token = %" PRIu64, event->fd, event->token);
 838
 839 error:
 840         return ret;
 841 }
 842
 843 static struct process_attr_tracker *
 844 _kernel_get_process_attr_tracker(struct ltt_kernel_session *session,
 845                                  enum lttng_process_attr process_attr)
 846 {
 847         switch (process_attr) {
 848         case LTTNG_PROCESS_ATTR_PROCESS_ID:
 849                 return session->tracker_pid;
 850         case LTTNG_PROCESS_ATTR_VIRTUAL_PROCESS_ID:
 851                 return session->tracker_vpid;
 852         case LTTNG_PROCESS_ATTR_USER_ID:
 853                 return session->tracker_uid;
 854         case LTTNG_PROCESS_ATTR_VIRTUAL_USER_ID:
 855                 return session->tracker_vuid;
 856         case LTTNG_PROCESS_ATTR_GROUP_ID:
 857                 return session->tracker_gid;
 858         case LTTNG_PROCESS_ATTR_VIRTUAL_GROUP_ID:
 859                 return session->tracker_vgid;
 860         default:
 861                 return nullptr;
 862         }
 863 }
 864
 865 const struct process_attr_tracker *
 866 kernel_get_process_attr_tracker(struct ltt_kernel_session *session,
 867                                 enum lttng_process_attr process_attr)
 868 {
 869         return (const struct process_attr_tracker *) _kernel_get_process_attr_tracker(session,
 870                                                                                       process_attr);
 871 }
 872
 873 enum lttng_error_code
 874 kernel_process_attr_tracker_set_tracking_policy(struct ltt_kernel_session *session,
 875                                                 enum lttng_process_attr process_attr,
 876                                                 enum lttng_tracking_policy policy)
 877 {
 878         int ret;
 879         enum lttng_error_code ret_code = LTTNG_OK;
 880         struct process_attr_tracker *tracker =
 881                 _kernel_get_process_attr_tracker(session, process_attr);
 882         enum lttng_tracking_policy previous_policy;
 883
 884         if (!tracker) {
 885                 ret_code = LTTNG_ERR_INVALID;
 886                 goto end;
 887         }
 888
 889         previous_policy = process_attr_tracker_get_tracking_policy(tracker);
 890         ret = process_attr_tracker_set_tracking_policy(tracker, policy);
 891         if (ret) {
 892                 ret_code = LTTNG_ERR_UNK;
 893                 goto end;
 894         }
 895
 896         if (previous_policy == policy) {
 897                 goto end;
 898         }
 899
 900         switch (policy) {
 901         case LTTNG_TRACKING_POLICY_INCLUDE_ALL:
 902                 if (process_attr == LTTNG_PROCESS_ATTR_PROCESS_ID) {
 903                         /*
 904                          * Maintain a special case for the process ID process
 905                          * attribute tracker as it was the only supported
 906                          * attribute prior to 2.12.
 907                          */
 908                         ret = kernctl_track_pid(session->fd, -1);
 909                 } else {
 910                         ret = kernctl_track_id(session->fd, process_attr, -1);
 911                 }
 912                 break;
 913         case LTTNG_TRACKING_POLICY_EXCLUDE_ALL:
 914         case LTTNG_TRACKING_POLICY_INCLUDE_SET:
 915                 /* fall-through. */
 916                 if (process_attr == LTTNG_PROCESS_ATTR_PROCESS_ID) {
 917                         /*
 918                          * Maintain a special case for the process ID process
 919                          * attribute tracker as it was the only supported
 920                          * attribute prior to 2.12.
 921                          */
 922                         ret = kernctl_untrack_pid(session->fd, -1);
 923                 } else {
 924                         ret = kernctl_untrack_id(session->fd, process_attr, -1);
 925                 }
 926                 break;
 927         default:
 928                 abort();
 929         }
 930         /* kern-ctl error handling */
 931         switch (-ret) {
 932         case 0:
 933                 ret_code = LTTNG_OK;
 934                 break;
 935         case EINVAL:
 936                 ret_code = LTTNG_ERR_INVALID;
 937                 break;
 938         case ENOMEM:
 939                 ret_code = LTTNG_ERR_NOMEM;
 940                 break;
 941         case EEXIST:
 942                 ret_code = LTTNG_ERR_PROCESS_ATTR_EXISTS;
 943                 break;
 944         default:
 945                 ret_code = LTTNG_ERR_UNK;
 946                 break;
 947         }
 948 end:
 949         return ret_code;
 950 }
 951
 952 enum lttng_error_code
 953 kernel_process_attr_tracker_inclusion_set_add_value(struct ltt_kernel_session *session,
 954                                                     enum lttng_process_attr process_attr,
 955                                                     const struct process_attr_value *value)
 956 {
 957         int ret, integral_value;
 958         enum lttng_error_code ret_code;
 959         struct process_attr_tracker *tracker;
 960         enum process_attr_tracker_status status;
 961
 962         /*
 963          * Convert process attribute tracker value to the integral
 964          * representation required by the kern-ctl API.
 965          */
 966         switch (process_attr) {
 967         case LTTNG_PROCESS_ATTR_PROCESS_ID:
 968         case LTTNG_PROCESS_ATTR_VIRTUAL_PROCESS_ID:
 969                 integral_value = (int) value->value.pid;
 970                 break;
 971         case LTTNG_PROCESS_ATTR_USER_ID:
 972         case LTTNG_PROCESS_ATTR_VIRTUAL_USER_ID:
 973                 if (value->type == LTTNG_PROCESS_ATTR_VALUE_TYPE_USER_NAME) {
 974                         uid_t uid;
 975
 976                         ret_code = utils_user_id_from_name(value->value.user_name, &uid);
 977                         if (ret_code != LTTNG_OK) {
 978                                 goto end;
 979                         }
 980                         integral_value = (int) uid;
 981                 } else {
 982                         integral_value = (int) value->value.uid;
 983                 }
 984                 break;
 985         case LTTNG_PROCESS_ATTR_GROUP_ID:
 986         case LTTNG_PROCESS_ATTR_VIRTUAL_GROUP_ID:
 987                 if (value->type == LTTNG_PROCESS_ATTR_VALUE_TYPE_GROUP_NAME) {
 988                         gid_t gid;
 989
 990                         ret_code = utils_group_id_from_name(value->value.group_name, &gid);
 991                         if (ret_code != LTTNG_OK) {
 992                                 goto end;
 993                         }
 994                         integral_value = (int) gid;
 995                 } else {
 996                         integral_value = (int) value->value.gid;
 997                 }
 998                 break;
 999         default:
1000                 ret_code = LTTNG_ERR_INVALID;
1001                 goto end;
1002         }
1003
1004         tracker = _kernel_get_process_attr_tracker(session, process_attr);
1005         if (!tracker) {
1006                 ret_code = LTTNG_ERR_INVALID;
1007                 goto end;
1008         }
1009
1010         status = process_attr_tracker_inclusion_set_add_value(tracker, value);
1011         if (status != PROCESS_ATTR_TRACKER_STATUS_OK) {
1012                 switch (status) {
1013                 case PROCESS_ATTR_TRACKER_STATUS_EXISTS:
1014                         ret_code = LTTNG_ERR_PROCESS_ATTR_EXISTS;
1015                         break;
1016                 case PROCESS_ATTR_TRACKER_STATUS_INVALID_TRACKING_POLICY:
1017                         ret_code = LTTNG_ERR_PROCESS_ATTR_TRACKER_INVALID_TRACKING_POLICY;
1018                         break;
1019                 case PROCESS_ATTR_TRACKER_STATUS_ERROR:
1020                 default:
1021                         ret_code = LTTNG_ERR_UNK;
1022                         break;
1023                 }
1024                 goto end;
1025         }
1026
1027         DBG("Kernel track %s %d for session id %" PRIu64,
1028             lttng_process_attr_to_string(process_attr),
1029             integral_value,
1030             session->id);
1031         if (process_attr == LTTNG_PROCESS_ATTR_PROCESS_ID) {
1032                 /*
1033                  * Maintain a special case for the process ID process attribute
1034                  * tracker as it was the only supported attribute prior to 2.12.
1035                  */
1036                 ret = kernctl_track_pid(session->fd, integral_value);
1037         } else {
1038                 ret = kernctl_track_id(session->fd, process_attr, integral_value);
1039         }
1040         if (ret == 0) {
1041                 ret_code = LTTNG_OK;
1042                 goto end;
1043         }
1044
1045         kernel_wait_quiescent();
1046
1047         /* kern-ctl error handling */
1048         switch (-ret) {
1049         case 0:
1050                 ret_code = LTTNG_OK;
1051                 break;
1052         case EINVAL:
1053                 ret_code = LTTNG_ERR_INVALID;
1054                 break;
1055         case ENOMEM:
1056                 ret_code = LTTNG_ERR_NOMEM;
1057                 break;
1058         case EEXIST:
1059                 ret_code = LTTNG_ERR_PROCESS_ATTR_EXISTS;
1060                 break;
1061         default:
1062                 ret_code = LTTNG_ERR_UNK;
1063                 break;
1064         }
1065
1066         /* Attempt to remove the value from the tracker. */
1067         status = process_attr_tracker_inclusion_set_remove_value(tracker, value);
1068         if (status != PROCESS_ATTR_TRACKER_STATUS_OK) {
1069                 ERR("Failed to roll-back the tracking of kernel %s process attribute %d while handling a kern-ctl error",
1070                     lttng_process_attr_to_string(process_attr),
1071                     integral_value);
1072         }
1073 end:
1074         return ret_code;
1075 }
1076
1077 enum lttng_error_code
1078 kernel_process_attr_tracker_inclusion_set_remove_value(struct ltt_kernel_session *session,
1079                                                        enum lttng_process_attr process_attr,
1080                                                        const struct process_attr_value *value)
1081 {
1082         int ret, integral_value;
1083         enum lttng_error_code ret_code;
1084         struct process_attr_tracker *tracker;
1085         enum process_attr_tracker_status status;
1086
1087         /*
1088          * Convert process attribute tracker value to the integral
1089          * representation required by the kern-ctl API.
1090          */
1091         switch (process_attr) {
1092         case LTTNG_PROCESS_ATTR_PROCESS_ID:
1093         case LTTNG_PROCESS_ATTR_VIRTUAL_PROCESS_ID:
1094                 integral_value = (int) value->value.pid;
1095                 break;
1096         case LTTNG_PROCESS_ATTR_USER_ID:
1097         case LTTNG_PROCESS_ATTR_VIRTUAL_USER_ID:
1098                 if (value->type == LTTNG_PROCESS_ATTR_VALUE_TYPE_USER_NAME) {
1099                         uid_t uid;
1100
1101                         ret_code = utils_user_id_from_name(value->value.user_name, &uid);
1102                         if (ret_code != LTTNG_OK) {
1103                                 goto end;
1104                         }
1105                         integral_value = (int) uid;
1106                 } else {
1107                         integral_value = (int) value->value.uid;
1108                 }
1109                 break;
1110         case LTTNG_PROCESS_ATTR_GROUP_ID:
1111         case LTTNG_PROCESS_ATTR_VIRTUAL_GROUP_ID:
1112                 if (value->type == LTTNG_PROCESS_ATTR_VALUE_TYPE_GROUP_NAME) {
1113                         gid_t gid;
1114
1115                         ret_code = utils_group_id_from_name(value->value.group_name, &gid);
1116                         if (ret_code != LTTNG_OK) {
1117                                 goto end;
1118                         }
1119                         integral_value = (int) gid;
1120                 } else {
1121                         integral_value = (int) value->value.gid;
1122                 }
1123                 break;
1124         default:
1125                 ret_code = LTTNG_ERR_INVALID;
1126                 goto end;
1127         }
1128
1129         tracker = _kernel_get_process_attr_tracker(session, process_attr);
1130         if (!tracker) {
1131                 ret_code = LTTNG_ERR_INVALID;
1132                 goto end;
1133         }
1134
1135         status = process_attr_tracker_inclusion_set_remove_value(tracker, value);
1136         if (status != PROCESS_ATTR_TRACKER_STATUS_OK) {
1137                 switch (status) {
1138                 case PROCESS_ATTR_TRACKER_STATUS_MISSING:
1139                         ret_code = LTTNG_ERR_PROCESS_ATTR_MISSING;
1140                         break;
1141                 case PROCESS_ATTR_TRACKER_STATUS_INVALID_TRACKING_POLICY:
1142                         ret_code = LTTNG_ERR_PROCESS_ATTR_TRACKER_INVALID_TRACKING_POLICY;
1143                         break;
1144                 case PROCESS_ATTR_TRACKER_STATUS_ERROR:
1145                 default:
1146                         ret_code = LTTNG_ERR_UNK;
1147                         break;
1148                 }
1149                 goto end;
1150         }
1151
1152         DBG("Kernel track %s %d for session id %" PRIu64,
1153             lttng_process_attr_to_string(process_attr),
1154             integral_value,
1155             session->id);
1156         if (process_attr == LTTNG_PROCESS_ATTR_PROCESS_ID) {
1157                 /*
1158                  * Maintain a special case for the process ID process attribute
1159                  * tracker as it was the only supported attribute prior to 2.12.
1160                  */
1161                 ret = kernctl_untrack_pid(session->fd, integral_value);
1162         } else {
1163                 ret = kernctl_untrack_id(session->fd, process_attr, integral_value);
1164         }
1165         if (ret == 0) {
1166                 ret_code = LTTNG_OK;
1167                 goto end;
1168         }
1169         kernel_wait_quiescent();
1170
1171         /* kern-ctl error handling */
1172         switch (-ret) {
1173         case 0:
1174                 ret_code = LTTNG_OK;
1175                 break;
1176         case EINVAL:
1177                 ret_code = LTTNG_ERR_INVALID;
1178                 break;
1179         case ENOMEM:
1180                 ret_code = LTTNG_ERR_NOMEM;
1181                 break;
1182         case ENOENT:
1183                 ret_code = LTTNG_ERR_PROCESS_ATTR_MISSING;
1184                 break;
1185         default:
1186                 ret_code = LTTNG_ERR_UNK;
1187                 break;
1188         }
1189
1190         /* Attempt to add the value to the tracker. */
1191         status = process_attr_tracker_inclusion_set_add_value(tracker, value);
1192         if (status != PROCESS_ATTR_TRACKER_STATUS_OK) {
1193                 ERR("Failed to roll-back the tracking of kernel %s process attribute %d while handling a kern-ctl error",
1194                     lttng_process_attr_to_string(process_attr),
1195                     integral_value);
1196         }
1197 end:
1198         return ret_code;
1199 }
1200
1201 /*
1202  * Create kernel metadata, open from the kernel tracer and add it to the
1203  * kernel session.
1204  */
1205 int kernel_open_metadata(struct ltt_kernel_session *session)
1206 {
1207         int ret;
1208         struct ltt_kernel_metadata *lkm = nullptr;
1209
1210         LTTNG_ASSERT(session);
1211
1212         /* Allocate kernel metadata */
1213         lkm = trace_kernel_create_metadata();
1214         if (lkm == nullptr) {
1215                 goto error;
1216         }
1217
1218         /* Kernel tracer metadata creation */
1219         ret = kernctl_open_metadata(session->fd, &lkm->conf->attr);
1220         if (ret < 0) {
1221                 goto error_open;
1222         }
1223
1224         lkm->fd = ret;
1225         lkm->key = ++next_kernel_channel_key;
1226         /* Prevent fd duplication after execlp() */
1227         ret = fcntl(lkm->fd, F_SETFD, FD_CLOEXEC);
1228         if (ret < 0) {
1229                 PERROR("fcntl session fd");
1230         }
1231
1232         session->metadata = lkm;
1233
1234         DBG("Kernel metadata opened (fd: %d)", lkm->fd);
1235
1236         return 0;
1237
1238 error_open:
1239         trace_kernel_destroy_metadata(lkm);
1240 error:
1241         return -1;
1242 }
1243
1244 /*
1245  * Start tracing session.
1246  */
1247 int kernel_start_session(struct ltt_kernel_session *session)
1248 {
1249         int ret;
1250
1251         LTTNG_ASSERT(session);
1252
1253         ret = kernctl_start_session(session->fd);
1254         if (ret < 0) {
1255                 PERROR("ioctl start session");
1256                 goto error;
1257         }
1258
1259         DBG("Kernel session started");
1260
1261         return 0;
1262
1263 error:
1264         return ret;
1265 }
1266
1267 /*
1268  * Make a kernel wait to make sure in-flight probe have completed.
1269  */
1270 void kernel_wait_quiescent()
1271 {
1272         int ret;
1273         int fd = kernel_tracer_fd;
1274
1275         DBG("Kernel quiescent wait on %d", fd);
1276
1277         ret = kernctl_wait_quiescent(fd);
1278         if (ret < 0) {
1279                 PERROR("wait quiescent ioctl");
1280                 ERR("Kernel quiescent wait failed");
1281         }
1282 }
1283
1284 /*
1285  *  Force flush buffer of metadata.
1286  */
1287 int kernel_metadata_flush_buffer(int fd)
1288 {
1289         int ret;
1290
1291         DBG("Kernel flushing metadata buffer on fd %d", fd);
1292
1293         ret = kernctl_buffer_flush(fd);
1294         if (ret < 0) {
1295                 ERR("Fail to flush metadata buffers %d (ret: %d)", fd, ret);
1296         }
1297
1298         return 0;
1299 }
1300
1301 /*
1302  * Force flush buffer for channel.
1303  */
1304 int kernel_flush_buffer(struct ltt_kernel_channel *channel)
1305 {
1306         int ret;
1307         struct ltt_kernel_stream *stream;
1308
1309         LTTNG_ASSERT(channel);
1310
1311         DBG("Flush buffer for channel %s", channel->channel->name);
1312
1313         cds_list_for_each_entry (stream, &channel->stream_list.head, list) {
1314                 DBG("Flushing channel stream %d", stream->fd);
1315                 ret = kernctl_buffer_flush(stream->fd);
1316                 if (ret < 0) {
1317                         PERROR("ioctl");
1318                         ERR("Fail to flush buffer for stream %d (ret: %d)", stream->fd, ret);
1319                 }
1320         }
1321
1322         return 0;
1323 }
1324
1325 /*
1326  * Stop tracing session.
1327  */
1328 int kernel_stop_session(struct ltt_kernel_session *session)
1329 {
1330         int ret;
1331
1332         LTTNG_ASSERT(session);
1333
1334         ret = kernctl_stop_session(session->fd);
1335         if (ret < 0) {
1336                 goto error;
1337         }
1338
1339         DBG("Kernel session stopped");
1340
1341         return 0;
1342
1343 error:
1344         return ret;
1345 }
1346
1347 /*
1348  * Open stream of channel, register it to the kernel tracer and add it
1349  * to the stream list of the channel.
1350  *
1351  * Note: given that the streams may appear in random order wrt CPU
1352  * number (e.g. cpu hotplug), the index value of the stream number in
1353  * the stream name is not necessarily linked to the CPU number.
1354  *
1355  * Return the number of created stream. Else, a negative value.
1356  */
1357 int kernel_open_channel_stream(struct ltt_kernel_channel *channel)
1358 {
1359         int ret;
1360         struct ltt_kernel_stream *lks;
1361
1362         LTTNG_ASSERT(channel);
1363
1364         while ((ret = kernctl_create_stream(channel->fd)) >= 0) {
1365                 lks = trace_kernel_create_stream(channel->channel->name, channel->stream_count);
1366                 if (lks == nullptr) {
1367                         ret = close(ret);
1368                         if (ret) {
1369                                 PERROR("close");
1370                         }
1371                         goto error;
1372                 }
1373
1374                 lks->fd = ret;
1375                 /* Prevent fd duplication after execlp() */
1376                 ret = fcntl(lks->fd, F_SETFD, FD_CLOEXEC);
1377                 if (ret < 0) {
1378                         PERROR("fcntl session fd");
1379                 }
1380
1381                 lks->tracefile_size = channel->channel->attr.tracefile_size;
1382                 lks->tracefile_count = channel->channel->attr.tracefile_count;
1383
1384                 /* Add stream to channel stream list */
1385                 cds_list_add(&lks->list, &channel->stream_list.head);
1386                 channel->stream_count++;
1387
1388                 DBG("Kernel stream %s created (fd: %d, state: %d)", lks->name, lks->fd, lks->state);
1389         }
1390
1391         return channel->stream_count;
1392
1393 error:
1394         return -1;
1395 }
1396
1397 /*
1398  * Open the metadata stream and set it to the kernel session.
1399  */
1400 int kernel_open_metadata_stream(struct ltt_kernel_session *session)
1401 {
1402         int ret;
1403
1404         LTTNG_ASSERT(session);
1405
1406         ret = kernctl_create_stream(session->metadata->fd);
1407         if (ret < 0) {
1408                 PERROR("kernel create metadata stream");
1409                 goto error;
1410         }
1411
1412         DBG("Kernel metadata stream created (fd: %d)", ret);
1413         session->metadata_stream_fd = ret;
1414         /* Prevent fd duplication after execlp() */
1415         ret = fcntl(session->metadata_stream_fd, F_SETFD, FD_CLOEXEC);
1416         if (ret < 0) {
1417                 PERROR("fcntl session fd");
1418         }
1419
1420         return 0;
1421
1422 error:
1423         return -1;
1424 }
1425
1426 /*
1427  * Get the event list from the kernel tracer and return the number of elements.
1428  */
1429 ssize_t kernel_list_events(struct lttng_event **events)
1430 {
1431         int fd, ret;
1432         char *event;
1433         size_t nbmem, count = 0;
1434         FILE *fp;
1435         struct lttng_event *elist;
1436
1437         LTTNG_ASSERT(events);
1438
1439         fd = kernctl_tracepoint_list(kernel_tracer_fd);
1440         if (fd < 0) {
1441                 PERROR("kernel tracepoint list");
1442                 goto error;
1443         }
1444
1445         fp = fdopen(fd, "r");
1446         if (fp == nullptr) {
1447                 PERROR("kernel tracepoint list fdopen");
1448                 goto error_fp;
1449         }
1450
1451         /*
1452          * Init memory size counter
1453          * See kernel-ctl.h for explanation of this value
1454          */
1455         nbmem = KERNEL_EVENT_INIT_LIST_SIZE;
1456         elist = calloc<lttng_event>(nbmem);
1457         if (elist == nullptr) {
1458                 PERROR("alloc list events");
1459                 count = -ENOMEM;
1460                 goto end;
1461         }
1462
1463         while (fscanf(fp, "event { name = %m[^;]; };\n", &event) == 1) {
1464                 if (count >= nbmem) {
1465                         struct lttng_event *new_elist;
1466                         size_t new_nbmem;
1467
1468                         new_nbmem = nbmem << 1;
1469                         DBG("Reallocating event list from %zu to %zu bytes", nbmem, new_nbmem);
1470                         new_elist = (lttng_event *) realloc(elist,
1471                                                             new_nbmem * sizeof(struct lttng_event));
1472                         if (new_elist == nullptr) {
1473                                 PERROR("realloc list events");
1474                                 free(event);
1475                                 free(elist);
1476                                 count = -ENOMEM;
1477                                 goto end;
1478                         }
1479                         /* Zero the new memory */
1480                         memset(new_elist + nbmem,
1481                                0,
1482                                (new_nbmem - nbmem) * sizeof(struct lttng_event));
1483                         nbmem = new_nbmem;
1484                         elist = new_elist;
1485                 }
1486                 strncpy(elist[count].name, event, LTTNG_SYMBOL_NAME_LEN);
1487                 elist[count].name[LTTNG_SYMBOL_NAME_LEN - 1] = '\0';
1488                 elist[count].enabled = -1;
1489                 count++;
1490                 free(event);
1491         }
1492
1493         *events = elist;
1494         DBG("Kernel list events done (%zu events)", count);
1495 end:
1496         ret = fclose(fp); /* closes both fp and fd */
1497         if (ret) {
1498                 PERROR("fclose");
1499         }
1500         return count;
1501
1502 error_fp:
1503         ret = close(fd);
1504         if (ret) {
1505                 PERROR("close");
1506         }
1507 error:
1508         return -1;
1509 }
1510
1511 /*
1512  * Get kernel version and validate it.
1513  */
1514 int kernel_validate_version(struct lttng_kernel_abi_tracer_version *version,
1515                             struct lttng_kernel_abi_tracer_abi_version *abi_version)
1516 {
1517         int ret;
1518
1519         ret = kernctl_tracer_version(kernel_tracer_fd, version);
1520         if (ret < 0) {
1521                 ERR("Failed to retrieve the lttng-modules version");
1522                 goto error;
1523         }
1524
1525         /* Validate version */
1526         if (version->major != VERSION_MAJOR) {
1527                 ERR("Kernel tracer major version (%d) is not compatible with lttng-tools major version (%d)",
1528                     version->major,
1529                     VERSION_MAJOR);
1530                 goto error_version;
1531         }
1532         ret = kernctl_tracer_abi_version(kernel_tracer_fd, abi_version);
1533         if (ret < 0) {
1534                 ERR("Failed to retrieve lttng-modules ABI version");
1535                 goto error;
1536         }
1537         if (abi_version->major != LTTNG_KERNEL_ABI_MAJOR_VERSION) {
1538                 ERR("Kernel tracer ABI version (%d.%d) does not match the expected ABI major version (%d.*)",
1539                     abi_version->major,
1540                     abi_version->minor,
1541                     LTTNG_KERNEL_ABI_MAJOR_VERSION);
1542                 goto error;
1543         }
1544         DBG2("Kernel tracer version validated (%d.%d, ABI %d.%d)",
1545              version->major,
1546              version->minor,
1547              abi_version->major,
1548              abi_version->minor);
1549         return 0;
1550
1551 error_version:
1552         ret = -1;
1553
1554 error:
1555         ERR("Kernel tracer version check failed; kernel tracing will not be available");
1556         return ret;
1557 }
1558
1559 /*
1560  * Kernel work-arounds called at the start of sessiond main().
1561  */
1562 int init_kernel_workarounds()
1563 {
1564         int ret;
1565         FILE *fp;
1566
1567         /*
1568          * boot_id needs to be read once before being used concurrently
1569          * to deal with a Linux kernel race. A fix is proposed for
1570          * upstream, but the work-around is needed for older kernels.
1571          */
1572         fp = fopen("/proc/sys/kernel/random/boot_id", "r");
1573         if (!fp) {
1574                 goto end_boot_id;
1575         }
1576         while (!feof(fp)) {
1577                 char buf[37] = "";
1578
1579                 ret = fread(buf, 1, sizeof(buf), fp);
1580                 if (ret < 0) {
1581                         /* Ignore error, we don't really care */
1582                 }
1583         }
1584         ret = fclose(fp);
1585         if (ret) {
1586                 PERROR("fclose");
1587         }
1588 end_boot_id:
1589         return 0;
1590 }
1591
1592 /*
1593  * Teardown of a kernel session, keeping data required by destroy notifiers.
1594  */
1595 void kernel_destroy_session(struct ltt_kernel_session *ksess)
1596 {
1597         struct lttng_trace_chunk *trace_chunk;
1598
1599         if (ksess == nullptr) {
1600                 DBG3("No kernel session when tearing down session");
1601                 return;
1602         }
1603
1604         DBG("Tearing down kernel session");
1605         trace_chunk = ksess->current_trace_chunk;
1606
1607         /*
1608          * Destroy channels on the consumer if at least one FD has been sent and we
1609          * are in no output mode because the streams are in *no* monitor mode so we
1610          * have to send a command to clean them up or else they leaked.
1611          */
1612         if (!ksess->output_traces && ksess->consumer_fds_sent) {
1613                 int ret;
1614                 struct consumer_socket *socket;
1615                 struct lttng_ht_iter iter;
1616
1617                 /* For each consumer socket. */
1618                 lttng::urcu::read_lock_guard read_lock;
1619
1620                 cds_lfht_for_each_entry (
1621                         ksess->consumer->socks->ht, &iter.iter, socket, node.node) {
1622                         struct ltt_kernel_channel *chan;
1623
1624                         /* For each channel, ask the consumer to destroy it. */
1625                         cds_list_for_each_entry (chan, &ksess->channel_list.head, list) {
1626                                 ret = kernel_consumer_destroy_channel(socket, chan);
1627                                 if (ret < 0) {
1628                                         /* Consumer is probably dead. Use next socket. */
1629                                         continue;
1630                                 }
1631                         }
1632                 }
1633         }
1634
1635         /* Close any relayd session */
1636         consumer_output_send_destroy_relayd(ksess->consumer);
1637
1638         trace_kernel_destroy_session(ksess);
1639         lttng_trace_chunk_put(trace_chunk);
1640 }
1641
1642 /* Teardown of data required by destroy notifiers. */
1643 void kernel_free_session(struct ltt_kernel_session *ksess)
1644 {
1645         if (ksess == nullptr) {
1646                 return;
1647         }
1648         trace_kernel_free_session(ksess);
1649 }
1650
1651 /*
1652  * Destroy a kernel channel object. It does not do anything on the tracer side.
1653  */
1654 void kernel_destroy_channel(struct ltt_kernel_channel *kchan)
1655 {
1656         struct ltt_kernel_session *ksess = nullptr;
1657
1658         LTTNG_ASSERT(kchan);
1659         LTTNG_ASSERT(kchan->channel);
1660
1661         DBG3("Kernel destroy channel %s", kchan->channel->name);
1662
1663         /* Update channel count of associated session. */
1664         if (kchan->session) {
1665                 /* Keep pointer reference so we can update it after the destroy. */
1666                 ksess = kchan->session;
1667         }
1668
1669         trace_kernel_destroy_channel(kchan);
1670
1671         /*
1672          * At this point the kernel channel is not visible anymore. This is safe
1673          * since in order to work on a visible kernel session, the tracing session
1674          * lock (ltt_session.lock) MUST be acquired.
1675          */
1676         if (ksess) {
1677                 ksess->channel_count--;
1678         }
1679 }
1680
1681 /*
1682  * Take a snapshot for a given kernel session.
1683  *
1684  * Return LTTNG_OK on success or else return a LTTNG_ERR code.
1685  */
1686 enum lttng_error_code kernel_snapshot_record(struct ltt_kernel_session *ksess,
1687                                              const struct consumer_output *output,
1688                                              uint64_t nb_packets_per_stream)
1689 {
1690         int err, ret, saved_metadata_fd;
1691         enum lttng_error_code status = LTTNG_OK;
1692         struct consumer_socket *socket;
1693         struct lttng_ht_iter iter;
1694         struct ltt_kernel_metadata *saved_metadata;
1695         char *trace_path = nullptr;
1696         size_t consumer_path_offset = 0;
1697
1698         LTTNG_ASSERT(ksess);
1699         LTTNG_ASSERT(ksess->consumer);
1700         LTTNG_ASSERT(output);
1701
1702         DBG("Kernel snapshot record started");
1703
1704         /* Save current metadata since the following calls will change it. */
1705         saved_metadata = ksess->metadata;
1706         saved_metadata_fd = ksess->metadata_stream_fd;
1707
1708         ret = kernel_open_metadata(ksess);
1709         if (ret < 0) {
1710                 status = LTTNG_ERR_KERN_META_FAIL;
1711                 goto error;
1712         }
1713
1714         ret = kernel_open_metadata_stream(ksess);
1715         if (ret < 0) {
1716                 status = LTTNG_ERR_KERN_META_FAIL;
1717                 goto error_open_stream;
1718         }
1719
1720         trace_path = setup_channel_trace_path(ksess->consumer, "", &consumer_path_offset);
1721         if (!trace_path) {
1722                 status = LTTNG_ERR_INVALID;
1723                 goto error;
1724         }
1725
1726         {
1727                 /* Send metadata to consumer and snapshot everything. */
1728                 lttng::urcu::read_lock_guard read_lock;
1729
1730                 cds_lfht_for_each_entry (output->socks->ht, &iter.iter, socket, node.node) {
1731                         struct ltt_kernel_channel *chan;
1732
1733                         pthread_mutex_lock(socket->lock);
1734                         /* This stream must not be monitored by the consumer. */
1735                         ret = kernel_consumer_add_metadata(socket, ksess, 0);
1736                         pthread_mutex_unlock(socket->lock);
1737                         if (ret < 0) {
1738                                 status = LTTNG_ERR_KERN_META_FAIL;
1739                                 goto error_consumer;
1740                         }
1741
1742                         /* For each channel, ask the consumer to snapshot it. */
1743                         cds_list_for_each_entry (chan, &ksess->channel_list.head, list) {
1744                                 status =
1745                                         consumer_snapshot_channel(socket,
1746                                                                   chan->key,
1747                                                                   output,
1748                                                                   0,
1749                                                                   &trace_path[consumer_path_offset],
1750                                                                   nb_packets_per_stream);
1751                                 if (status != LTTNG_OK) {
1752                                         (void) kernel_consumer_destroy_metadata(socket,
1753                                                                                 ksess->metadata);
1754                                         goto error_consumer;
1755                                 }
1756                         }
1757
1758                         /* Snapshot metadata, */
1759                         status = consumer_snapshot_channel(socket,
1760                                                            ksess->metadata->key,
1761                                                            output,
1762                                                            1,
1763                                                            &trace_path[consumer_path_offset],
1764                                                            0);
1765                         if (status != LTTNG_OK) {
1766                                 goto error_consumer;
1767                         }
1768
1769                         /*
1770                          * The metadata snapshot is done, ask the consumer to destroy it since
1771                          * it's not monitored on the consumer side.
1772                          */
1773                         (void) kernel_consumer_destroy_metadata(socket, ksess->metadata);
1774                 }
1775         }
1776
1777 error_consumer:
1778         /* Close newly opened metadata stream. It's now on the consumer side. */
1779         err = close(ksess->metadata_stream_fd);
1780         if (err < 0) {
1781                 PERROR("close snapshot kernel");
1782         }
1783
1784 error_open_stream:
1785         trace_kernel_destroy_metadata(ksess->metadata);
1786 error:
1787         /* Restore metadata state.*/
1788         ksess->metadata = saved_metadata;
1789         ksess->metadata_stream_fd = saved_metadata_fd;
1790         free(trace_path);
1791         return status;
1792 }
1793
1794 /*
1795  * Get the syscall mask array from the kernel tracer.
1796  *
1797  * Return 0 on success else a negative value. In both case, syscall_mask should
1798  * be freed.
1799  */
1800 int kernel_syscall_mask(int chan_fd, char **syscall_mask, uint32_t *nr_bits)
1801 {
1802         LTTNG_ASSERT(syscall_mask);
1803         LTTNG_ASSERT(nr_bits);
1804
1805         return kernctl_syscall_mask(chan_fd, syscall_mask, nr_bits);
1806 }
1807
1808 static int kernel_tracer_abi_greater_or_equal(unsigned int major, unsigned int minor)
1809 {
1810         int ret;
1811         struct lttng_kernel_abi_tracer_abi_version abi;
1812
1813         ret = kernctl_tracer_abi_version(kernel_tracer_fd, &abi);
1814         if (ret < 0) {
1815                 ERR("Failed to retrieve lttng-modules ABI version");
1816                 goto error;
1817         }
1818
1819         ret = abi.major > major || (abi.major == major && abi.minor >= minor);
1820 error:
1821         return ret;
1822 }
1823
1824 /*
1825  * Check for the support of the RING_BUFFER_SNAPSHOT_SAMPLE_POSITIONS via abi
1826  * version number.
1827  *
1828  * Return 1 on success, 0 when feature is not supported, negative value in case
1829  * of errors.
1830  */
1831 int kernel_supports_ring_buffer_snapshot_sample_positions()
1832 {
1833         /*
1834          * RING_BUFFER_SNAPSHOT_SAMPLE_POSITIONS was introduced in 2.3
1835          */
1836         return kernel_tracer_abi_greater_or_equal(2, 3);
1837 }
1838
1839 /*
1840  * Check for the support of the packet sequence number via abi version number.
1841  *
1842  * Return 1 on success, 0 when feature is not supported, negative value in case
1843  * of errors.
1844  */
1845 int kernel_supports_ring_buffer_packet_sequence_number()
1846 {
1847         /*
1848          * Packet sequence number was introduced in LTTng 2.8,
1849          * lttng-modules ABI 2.1.
1850          */
1851         return kernel_tracer_abi_greater_or_equal(2, 1);
1852 }
1853
1854 /*
1855  * Check for the support of event notifiers via abi version number.
1856  *
1857  * Return 1 on success, 0 when feature is not supported, negative value in case
1858  * of errors.
1859  */
1860 int kernel_supports_event_notifiers()
1861 {
1862         /*
1863          * Event notifiers were introduced in LTTng 2.13, lttng-modules ABI 2.6.
1864          */
1865         return kernel_tracer_abi_greater_or_equal(2, 6);
1866 }
1867
1868 /*
1869  * Rotate a kernel session.
1870  *
1871  * Return LTTNG_OK on success or else an LTTng error code.
1872  */
1873 enum lttng_error_code kernel_rotate_session(struct ltt_session *session)
1874 {
1875         int ret;
1876         enum lttng_error_code status = LTTNG_OK;
1877         struct consumer_socket *socket;
1878         struct lttng_ht_iter iter;
1879         struct ltt_kernel_session *ksess = session->kernel_session;
1880
1881         LTTNG_ASSERT(ksess);
1882         LTTNG_ASSERT(ksess->consumer);
1883
1884         DBG("Rotate kernel session %s started (session %" PRIu64 ")", session->name, session->id);
1885
1886         {
1887                 /*
1888                  * Note that this loop will end after one iteration given that there is
1889                  * only one kernel consumer.
1890                  */
1891                 lttng::urcu::read_lock_guard read_lock;
1892
1893                 cds_lfht_for_each_entry (
1894                         ksess->consumer->socks->ht, &iter.iter, socket, node.node) {
1895                         struct ltt_kernel_channel *chan;
1896
1897                         /* For each channel, ask the consumer to rotate it. */
1898                         cds_list_for_each_entry (chan, &ksess->channel_list.head, list) {
1899                                 DBG("Rotate kernel channel %" PRIu64 ", session %s",
1900                                     chan->key,
1901                                     session->name);
1902                                 ret = consumer_rotate_channel(socket,
1903                                                               chan->key,
1904                                                               ksess->consumer,
1905                                                               /* is_metadata_channel */ false);
1906                                 if (ret < 0) {
1907                                         status = LTTNG_ERR_ROTATION_FAIL_CONSUMER;
1908                                         goto error;
1909                                 }
1910                         }
1911
1912                         /*
1913                          * Rotate the metadata channel.
1914                          */
1915                         ret = consumer_rotate_channel(socket,
1916                                                       ksess->metadata->key,
1917                                                       ksess->consumer,
1918                                                       /* is_metadata_channel */ true);
1919                         if (ret < 0) {
1920                                 status = LTTNG_ERR_ROTATION_FAIL_CONSUMER;
1921                                 goto error;
1922                         }
1923                 }
1924         }
1925
1926 error:
1927         return status;
1928 }
1929
1930 enum lttng_error_code kernel_create_channel_subdirectories(const struct ltt_kernel_session *ksess)
1931 {
1932         enum lttng_error_code ret = LTTNG_OK;
1933         enum lttng_trace_chunk_status chunk_status;
1934
1935         lttng::urcu::read_lock_guard read_lock;
1936         LTTNG_ASSERT(ksess->current_trace_chunk);
1937
1938         /*
1939          * Create the index subdirectory which will take care
1940          * of implicitly creating the channel's path.
1941          */
1942         chunk_status = lttng_trace_chunk_create_subdirectory(
1943                 ksess->current_trace_chunk, DEFAULT_KERNEL_TRACE_DIR "/" DEFAULT_INDEX_DIR);
1944         if (chunk_status != LTTNG_TRACE_CHUNK_STATUS_OK) {
1945                 ret = LTTNG_ERR_CREATE_DIR_FAIL;
1946                 goto error;
1947         }
1948 error:
1949         return ret;
1950 }
1951
1952 /*
1953  * Setup necessary data for kernel tracer action.
1954  */
1955 int init_kernel_tracer()
1956 {
1957         int ret;
1958         bool is_root = !getuid();
1959
1960         /* Modprobe lttng kernel modules */
1961         ret = modprobe_lttng_control();
1962         if (ret < 0) {
1963                 goto error;
1964         }
1965
1966         /* Open debugfs lttng */
1967         kernel_tracer_fd = open(module_proc_lttng, O_RDWR);
1968         if (kernel_tracer_fd < 0) {
1969                 DBG("Failed to open %s", module_proc_lttng);
1970                 goto error_open;
1971         }
1972
1973         /* Validate kernel version */
1974         ret = kernel_validate_version(&the_kernel_tracer_version, &the_kernel_tracer_abi_version);
1975         if (ret < 0) {
1976                 goto error_version;
1977         }
1978
1979         ret = modprobe_lttng_data();
1980         if (ret < 0) {
1981                 goto error_modules;
1982         }
1983
1984         ret = kernel_supports_ring_buffer_snapshot_sample_positions();
1985         if (ret < 0) {
1986                 goto error_modules;
1987         }
1988         if (ret < 1) {
1989                 WARN("Kernel tracer does not support buffer monitoring. "
1990                      "The monitoring timer of channels in the kernel domain "
1991                      "will be set to 0 (disabled).");
1992         }
1993
1994         ret = kernel_supports_event_notifiers();
1995         if (ret < 0) {
1996                 ERR("Failed to check for kernel tracer event notifier support");
1997                 goto error_modules;
1998         }
1999         ret = kernel_create_event_notifier_group(&kernel_tracer_event_notifier_group_fd);
2000         if (ret < 0) {
2001                 /* This is not fatal. */
2002                 WARN("Failed to create kernel event notifier group");
2003                 kernel_tracer_event_notifier_group_fd = -1;
2004         } else {
2005                 enum event_notifier_error_accounting_status error_accounting_status;
2006                 enum lttng_error_code error_code_ret =
2007                         kernel_create_event_notifier_group_notification_fd(
2008                                 &kernel_tracer_event_notifier_group_notification_fd);
2009
2010                 if (error_code_ret != LTTNG_OK) {
2011                         goto error_modules;
2012                 }
2013
2014                 error_accounting_status = event_notifier_error_accounting_register_kernel(
2015                         kernel_tracer_event_notifier_group_fd);
2016                 if (error_accounting_status != EVENT_NOTIFIER_ERROR_ACCOUNTING_STATUS_OK) {
2017                         ERR("Failed to initialize event notifier error accounting for kernel tracer");
2018                         error_code_ret = LTTNG_ERR_EVENT_NOTIFIER_ERROR_ACCOUNTING;
2019                         goto error_modules;
2020                 }
2021
2022                 kernel_token_to_event_notifier_rule_ht = cds_lfht_new(
2023                         DEFAULT_HT_SIZE, 1, 0, CDS_LFHT_AUTO_RESIZE | CDS_LFHT_ACCOUNTING, nullptr);
2024                 if (!kernel_token_to_event_notifier_rule_ht) {
2025                         goto error_token_ht;
2026                 }
2027         }
2028
2029         DBG("Kernel tracer initialized: kernel tracer fd = %d, event notifier group fd = %d, event notifier group notification fd = %d",
2030             kernel_tracer_fd,
2031             kernel_tracer_event_notifier_group_fd,
2032             kernel_tracer_event_notifier_group_notification_fd);
2033
2034         ret = syscall_init_table(kernel_tracer_fd);
2035         if (ret < 0) {
2036                 ERR("Unable to populate syscall table. Syscall tracing won't "
2037                     "work for this session daemon.");
2038         }
2039
2040         return 0;
2041
2042 error_version:
2043         modprobe_remove_lttng_control();
2044         ret = close(kernel_tracer_fd);
2045         if (ret) {
2046                 PERROR("Failed to close kernel tracer file descriptor: fd = %d", kernel_tracer_fd);
2047         }
2048
2049         kernel_tracer_fd = -1;
2050         return LTTNG_ERR_KERN_VERSION;
2051
2052 error_token_ht:
2053         ret = close(kernel_tracer_event_notifier_group_notification_fd);
2054         if (ret) {
2055                 PERROR("Failed to close kernel tracer event notifier group notification file descriptor: fd = %d",
2056                        kernel_tracer_event_notifier_group_notification_fd);
2057         }
2058
2059         kernel_tracer_event_notifier_group_notification_fd = -1;
2060
2061 error_modules:
2062         ret = close(kernel_tracer_event_notifier_group_fd);
2063         if (ret) {
2064                 PERROR("Failed to close kernel tracer event notifier group file descriptor: fd = %d",
2065                        kernel_tracer_event_notifier_group_fd);
2066         }
2067
2068         kernel_tracer_event_notifier_group_fd = -1;
2069
2070         ret = close(kernel_tracer_fd);
2071         if (ret) {
2072                 PERROR("Failed to close kernel tracer file descriptor: fd = %d", kernel_tracer_fd);
2073         }
2074
2075         kernel_tracer_fd = -1;
2076
2077 error_open:
2078         modprobe_remove_lttng_control();
2079
2080 error:
2081         WARN("No kernel tracer available");
2082         kernel_tracer_fd = -1;
2083         if (!is_root) {
2084                 return LTTNG_ERR_NEED_ROOT_SESSIOND;
2085         } else {
2086                 return LTTNG_ERR_KERN_NA;
2087         }
2088 }
2089
2090 void cleanup_kernel_tracer()
2091 {
2092         DBG2("Closing kernel event notifier group notification file descriptor");
2093         if (kernel_tracer_event_notifier_group_notification_fd >= 0) {
2094                 int ret = notification_thread_command_remove_tracer_event_source(
2095                         the_notification_thread_handle,
2096                         kernel_tracer_event_notifier_group_notification_fd);
2097                 if (ret != LTTNG_OK) {
2098                         ERR("Failed to remove kernel event notifier notification from notification thread");
2099                 }
2100
2101                 ret = close(kernel_tracer_event_notifier_group_notification_fd);
2102                 if (ret) {
2103                         PERROR("Failed to close kernel event notifier group notification file descriptor: fd = %d",
2104                                kernel_tracer_event_notifier_group_notification_fd);
2105                 }
2106
2107                 kernel_tracer_event_notifier_group_notification_fd = -1;
2108         }
2109
2110         if (kernel_token_to_event_notifier_rule_ht) {
2111                 const int ret = cds_lfht_destroy(kernel_token_to_event_notifier_rule_ht, nullptr);
2112                 LTTNG_ASSERT(ret == 0);
2113         }
2114
2115         DBG2("Closing kernel event notifier group file descriptor");
2116         if (kernel_tracer_event_notifier_group_fd >= 0) {
2117                 const int ret = close(kernel_tracer_event_notifier_group_fd);
2118
2119                 if (ret) {
2120                         PERROR("Failed to close kernel event notifier group file descriptor: fd = %d",
2121                                kernel_tracer_event_notifier_group_fd);
2122                 }
2123
2124                 kernel_tracer_event_notifier_group_fd = -1;
2125         }
2126
2127         DBG2("Closing kernel fd");
2128         if (kernel_tracer_fd >= 0) {
2129                 const int ret = close(kernel_tracer_fd);
2130
2131                 if (ret) {
2132                         PERROR("Failed to close kernel tracer file descriptor: fd = %d",
2133                                kernel_tracer_fd);
2134                 }
2135
2136                 kernel_tracer_fd = -1;
2137         }
2138
2139         free(syscall_table);
2140 }
2141
2142 bool kernel_tracer_is_initialized()
2143 {
2144         return kernel_tracer_fd >= 0;
2145 }
2146
2147 /*
2148  *  Clear a kernel session.
2149  *
2150  * Return LTTNG_OK on success or else an LTTng error code.
2151  */
2152 enum lttng_error_code kernel_clear_session(struct ltt_session *session)
2153 {
2154         int ret;
2155         enum lttng_error_code status = LTTNG_OK;
2156         struct consumer_socket *socket;
2157         struct lttng_ht_iter iter;
2158         struct ltt_kernel_session *ksess = session->kernel_session;
2159
2160         LTTNG_ASSERT(ksess);
2161         LTTNG_ASSERT(ksess->consumer);
2162
2163         DBG("Clear kernel session %s (session %" PRIu64 ")", session->name, session->id);
2164
2165         if (ksess->active) {
2166                 ERR("Expecting inactive session %s (%" PRIu64 ")", session->name, session->id);
2167                 status = LTTNG_ERR_FATAL;
2168                 goto end;
2169         }
2170
2171         {
2172                 /*
2173                  * Note that this loop will end after one iteration given that there is
2174                  * only one kernel consumer.
2175                  */
2176                 lttng::urcu::read_lock_guard read_lock;
2177
2178                 cds_lfht_for_each_entry (
2179                         ksess->consumer->socks->ht, &iter.iter, socket, node.node) {
2180                         struct ltt_kernel_channel *chan;
2181
2182                         /* For each channel, ask the consumer to clear it. */
2183                         cds_list_for_each_entry (chan, &ksess->channel_list.head, list) {
2184                                 DBG("Clear kernel channel %" PRIu64 ", session %s",
2185                                     chan->key,
2186                                     session->name);
2187                                 ret = consumer_clear_channel(socket, chan->key);
2188                                 if (ret < 0) {
2189                                         goto error;
2190                                 }
2191                         }
2192
2193                         if (!ksess->metadata) {
2194                                 /*
2195                                  * Nothing to do for the metadata.
2196                                  * This is a snapshot session.
2197                                  * The metadata is genererated on the fly.
2198                                  */
2199                                 continue;
2200                         }
2201
2202                         /*
2203                          * Clear the metadata channel.
2204                          * Metadata channel is not cleared per se but we still need to
2205                          * perform a rotation operation on it behind the scene.
2206                          */
2207                         ret = consumer_clear_channel(socket, ksess->metadata->key);
2208                         if (ret < 0) {
2209                                 goto error;
2210                         }
2211                 }
2212         }
2213
2214         goto end;
2215 error:
2216         switch (-ret) {
2217         case LTTCOMM_CONSUMERD_RELAYD_CLEAR_DISALLOWED:
2218                 status = LTTNG_ERR_CLEAR_RELAY_DISALLOWED;
2219                 break;
2220         default:
2221                 status = LTTNG_ERR_CLEAR_FAIL_CONSUMER;
2222                 break;
2223         }
2224 end:
2225         return status;
2226 }
2227
2228 enum lttng_error_code
2229 kernel_create_event_notifier_group_notification_fd(int *event_notifier_group_notification_fd)
2230 {
2231         int local_fd = -1, ret;
2232         enum lttng_error_code error_code_ret;
2233
2234         LTTNG_ASSERT(event_notifier_group_notification_fd);
2235
2236         ret = kernctl_create_event_notifier_group_notification_fd(
2237                 kernel_tracer_event_notifier_group_fd);
2238         if (ret < 0) {
2239                 PERROR("Failed to create kernel event notifier group notification file descriptor");
2240                 error_code_ret = LTTNG_ERR_EVENT_NOTIFIER_GROUP_NOTIFICATION_FD;
2241                 goto error;
2242         }
2243
2244         local_fd = ret;
2245
2246         /* Prevent fd duplication after execlp(). */
2247         ret = fcntl(local_fd, F_SETFD, FD_CLOEXEC);
2248         if (ret < 0) {
2249                 PERROR("Failed to set FD_CLOEXEC on kernel event notifier group notification file descriptor: fd = %d",
2250                        local_fd);
2251                 error_code_ret = LTTNG_ERR_EVENT_NOTIFIER_GROUP_NOTIFICATION_FD;
2252                 goto error;
2253         }
2254
2255         DBG("Created kernel notifier group notification file descriptor: fd = %d", local_fd);
2256         error_code_ret = LTTNG_OK;
2257         *event_notifier_group_notification_fd = local_fd;
2258         local_fd = -1;
2259
2260 error:
2261         if (local_fd >= 0) {
2262                 ret = close(local_fd);
2263                 if (ret) {
2264                         PERROR("Failed to close kernel event notifier group notification file descriptor: fd = %d",
2265                                local_fd);
2266                 }
2267         }
2268
2269         return error_code_ret;
2270 }
2271
2272 enum lttng_error_code
2273 kernel_destroy_event_notifier_group_notification_fd(int event_notifier_group_notification_fd)
2274 {
2275         enum lttng_error_code ret_code = LTTNG_OK;
2276
2277         DBG("Closing event notifier group notification file descriptor: fd = %d",
2278             event_notifier_group_notification_fd);
2279         if (event_notifier_group_notification_fd >= 0) {
2280                 const int ret = close(event_notifier_group_notification_fd);
2281                 if (ret) {
2282                         PERROR("Failed to close event notifier group notification file descriptor: fd = %d",
2283                                event_notifier_group_notification_fd);
2284                 }
2285         }
2286
2287         return ret_code;
2288 }
2289
2290 static unsigned long hash_trigger(const struct lttng_trigger *trigger)
2291 {
2292         const struct lttng_condition *condition = lttng_trigger_get_const_condition(trigger);
2293
2294         return lttng_condition_hash(condition);
2295 }
2296
2297 static int match_trigger(struct cds_lfht_node *node, const void *key)
2298 {
2299         const struct ltt_kernel_event_notifier_rule *event_notifier_rule;
2300         const struct lttng_trigger *trigger = (lttng_trigger *) key;
2301
2302         event_notifier_rule =
2303                 caa_container_of(node, const struct ltt_kernel_event_notifier_rule, ht_node);
2304
2305         return lttng_trigger_is_equal(trigger, event_notifier_rule->trigger);
2306 }
2307
2308 static enum lttng_error_code kernel_create_event_notifier_rule(
2309         struct lttng_trigger *trigger, const struct lttng_credentials *creds, uint64_t token)
2310 {
2311         int err, fd, ret = 0;
2312         enum lttng_error_code error_code_ret;
2313         enum lttng_condition_status condition_status;
2314         enum lttng_condition_type condition_type;
2315         enum lttng_event_rule_type event_rule_type;
2316         struct ltt_kernel_event_notifier_rule *event_notifier_rule;
2317         struct lttng_kernel_abi_event_notifier kernel_event_notifier = {};
2318         unsigned int capture_bytecode_count = 0, i;
2319         const struct lttng_condition *condition = nullptr;
2320         const struct lttng_event_rule *event_rule = nullptr;
2321         enum lttng_condition_status cond_status;
2322
2323         LTTNG_ASSERT(trigger);
2324
2325         condition = lttng_trigger_get_const_condition(trigger);
2326         LTTNG_ASSERT(condition);
2327
2328         condition_type = lttng_condition_get_type(condition);
2329         LTTNG_ASSERT(condition_type == LTTNG_CONDITION_TYPE_EVENT_RULE_MATCHES);
2330
2331         /* Does not acquire a reference. */
2332         condition_status = lttng_condition_event_rule_matches_get_rule(condition, &event_rule);
2333         LTTNG_ASSERT(condition_status == LTTNG_CONDITION_STATUS_OK);
2334         LTTNG_ASSERT(event_rule);
2335
2336         event_rule_type = lttng_event_rule_get_type(event_rule);
2337         LTTNG_ASSERT(event_rule_type != LTTNG_EVENT_RULE_TYPE_UNKNOWN);
2338
2339         error_code_ret = trace_kernel_create_event_notifier_rule(
2340                 trigger,
2341                 token,
2342                 lttng_condition_event_rule_matches_get_error_counter_index(condition),
2343                 &event_notifier_rule);
2344         if (error_code_ret != LTTNG_OK) {
2345                 goto error;
2346         }
2347
2348         error_code_ret = trace_kernel_init_event_notifier_from_event_rule(event_rule,
2349                                                                           &kernel_event_notifier);
2350         if (error_code_ret != LTTNG_OK) {
2351                 goto free_event;
2352         }
2353
2354         kernel_event_notifier.event.token = event_notifier_rule->token;
2355         kernel_event_notifier.error_counter_idx =
2356                 lttng_condition_event_rule_matches_get_error_counter_index(condition);
2357
2358         fd = kernctl_create_event_notifier(kernel_tracer_event_notifier_group_fd,
2359                                            &kernel_event_notifier);
2360         if (fd < 0) {
2361                 switch (-fd) {
2362                 case EEXIST:
2363                         error_code_ret = LTTNG_ERR_KERN_EVENT_EXIST;
2364                         break;
2365                 case ENOSYS:
2366                         WARN("Failed to create kernel event notifier: not notifier type not implemented");
2367                         error_code_ret = LTTNG_ERR_KERN_EVENT_ENOSYS;
2368                         break;
2369                 case ENOENT:
2370                         WARN("Failed to create kernel event notifier: not found: name = '%s'",
2371                              kernel_event_notifier.event.name);
2372                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2373                         break;
2374                 default:
2375                         PERROR("Failed to create kernel event notifier: error code = %d, name = '%s'",
2376                                fd,
2377                                kernel_event_notifier.event.name);
2378                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2379                 }
2380                 goto free_event;
2381         }
2382
2383         event_notifier_rule->fd = fd;
2384         /* Prevent fd duplication after execlp(). */
2385         err = fcntl(event_notifier_rule->fd, F_SETFD, FD_CLOEXEC);
2386         if (err < 0) {
2387                 PERROR("Failed to set FD_CLOEXEC on kernel event notifier file descriptor: fd = %d",
2388                        fd);
2389                 error_code_ret = LTTNG_ERR_FATAL;
2390                 goto set_cloexec_error;
2391         }
2392
2393         if (event_notifier_rule->filter) {
2394                 err = kernctl_filter(event_notifier_rule->fd, event_notifier_rule->filter);
2395                 if (err < 0) {
2396                         switch (-err) {
2397                         case ENOMEM:
2398                                 error_code_ret = LTTNG_ERR_FILTER_NOMEM;
2399                                 break;
2400                         default:
2401                                 error_code_ret = LTTNG_ERR_FILTER_INVAL;
2402                                 break;
2403                         }
2404                         goto filter_error;
2405                 }
2406         }
2407
2408         if (lttng_event_rule_get_type(event_rule) == LTTNG_EVENT_RULE_TYPE_KERNEL_UPROBE) {
2409                 ret = userspace_probe_event_rule_add_callsites(
2410                         event_rule, creds, event_notifier_rule->fd);
2411                 if (ret) {
2412                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2413                         goto add_callsite_error;
2414                 }
2415         }
2416
2417         /* Set the capture bytecode if any. */
2418         cond_status = lttng_condition_event_rule_matches_get_capture_descriptor_count(
2419                 condition, &capture_bytecode_count);
2420         LTTNG_ASSERT(cond_status == LTTNG_CONDITION_STATUS_OK);
2421
2422         for (i = 0; i < capture_bytecode_count; i++) {
2423                 const struct lttng_bytecode *capture_bytecode =
2424                         lttng_condition_event_rule_matches_get_capture_bytecode_at_index(condition,
2425                                                                                          i);
2426
2427                 if (capture_bytecode == nullptr) {
2428                         ERR("Unexpected NULL capture bytecode on condition");
2429                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2430                         goto capture_error;
2431                 }
2432
2433                 ret = kernctl_capture(event_notifier_rule->fd, capture_bytecode);
2434                 if (ret < 0) {
2435                         ERR("Failed to set capture bytecode on event notifier rule fd: fd = %d",
2436                             event_notifier_rule->fd);
2437                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2438                         goto capture_error;
2439                 }
2440         }
2441
2442         err = kernctl_enable(event_notifier_rule->fd);
2443         if (err < 0) {
2444                 switch (-err) {
2445                 case EEXIST:
2446                         error_code_ret = LTTNG_ERR_KERN_EVENT_EXIST;
2447                         break;
2448                 default:
2449                         PERROR("enable kernel event notifier");
2450                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2451                         break;
2452                 }
2453                 goto enable_error;
2454         }
2455
2456         /* Add trigger to kernel token mapping in the hash table. */
2457         {
2458                 lttng::urcu::read_lock_guard read_lock;
2459                 cds_lfht_add(kernel_token_to_event_notifier_rule_ht,
2460                              hash_trigger(trigger),
2461                              &event_notifier_rule->ht_node);
2462         }
2463
2464         DBG("Created kernel event notifier: name = '%s', fd = %d",
2465             kernel_event_notifier.event.name,
2466             event_notifier_rule->fd);
2467
2468         return LTTNG_OK;
2469
2470 capture_error:
2471 add_callsite_error:
2472 enable_error:
2473 set_cloexec_error:
2474 filter_error:
2475 {
2476         const int close_ret = close(event_notifier_rule->fd);
2477
2478         if (close_ret) {
2479                 PERROR("Failed to close kernel event notifier file descriptor: fd = %d",
2480                        event_notifier_rule->fd);
2481         }
2482 }
2483 free_event:
2484         free(event_notifier_rule);
2485 error:
2486         return error_code_ret;
2487 }
2488
2489 enum lttng_error_code kernel_register_event_notifier(struct lttng_trigger *trigger,
2490                                                      const struct lttng_credentials *cmd_creds)
2491 {
2492         enum lttng_error_code ret;
2493         enum lttng_condition_status status;
2494         enum lttng_domain_type domain_type;
2495         const struct lttng_event_rule *event_rule;
2496         const struct lttng_condition *const condition = lttng_trigger_get_const_condition(trigger);
2497         const uint64_t token = lttng_trigger_get_tracer_token(trigger);
2498
2499         LTTNG_ASSERT(condition);
2500
2501         /* Does not acquire a reference to the event rule. */
2502         status = lttng_condition_event_rule_matches_get_rule(condition, &event_rule);
2503         LTTNG_ASSERT(status == LTTNG_CONDITION_STATUS_OK);
2504
2505         domain_type = lttng_event_rule_get_domain_type(event_rule);
2506         LTTNG_ASSERT(domain_type == LTTNG_DOMAIN_KERNEL);
2507
2508         ret = kernel_create_event_notifier_rule(trigger, cmd_creds, token);
2509         if (ret != LTTNG_OK) {
2510                 ERR("Failed to create kernel event notifier rule");
2511         }
2512
2513         return ret;
2514 }
2515
2516 enum lttng_error_code kernel_unregister_event_notifier(const struct lttng_trigger *trigger)
2517 {
2518         struct ltt_kernel_event_notifier_rule *token_event_rule_element;
2519         struct cds_lfht_node *node;
2520         struct cds_lfht_iter iter;
2521         enum lttng_error_code error_code_ret;
2522         int ret;
2523
2524         lttng::urcu::read_lock_guard read_lock;
2525
2526         cds_lfht_lookup(kernel_token_to_event_notifier_rule_ht,
2527                         hash_trigger(trigger),
2528                         match_trigger,
2529                         trigger,
2530                         &iter);
2531
2532         node = cds_lfht_iter_get_node(&iter);
2533         if (!node) {
2534                 error_code_ret = LTTNG_ERR_TRIGGER_NOT_FOUND;
2535                 goto error;
2536         }
2537
2538         token_event_rule_element =
2539                 caa_container_of(node, struct ltt_kernel_event_notifier_rule, ht_node);
2540
2541         ret = kernel_disable_event_notifier_rule(token_event_rule_element);
2542         if (ret) {
2543                 error_code_ret = LTTNG_ERR_FATAL;
2544                 goto error;
2545         }
2546
2547         trace_kernel_destroy_event_notifier_rule(token_event_rule_element);
2548         error_code_ret = LTTNG_OK;
2549
2550 error:
2551
2552         return error_code_ret;
2553 }
2554
2555 int kernel_get_notification_fd()
2556 {
2557         return kernel_tracer_event_notifier_group_notification_fd;
2558 }