src/bin/lttng-sessiond/kernel.c

   1 /*
   2  * Copyright (C) 2011 EfficiOS Inc.
   3  *
   4  * SPDX-License-Identifier: GPL-2.0-only
   5  *
   6  */
   7
   8 #define _LGPL_SOURCE
   9 #include <fcntl.h>
  10 #include <stdlib.h>
  11 #include <stdio.h>
  12 #include <string.h>
  13 #include <unistd.h>
  14 #include <inttypes.h>
  15 #include <sys/types.h>
  16
  17 #include <common/common.h>
  18 #include <common/hashtable/utils.h>
  19 #include <common/trace-chunk.h>
  20 #include <common/kernel-ctl/kernel-ctl.h>
  21 #include <common/kernel-ctl/kernel-ioctl.h>
  22 #include <common/sessiond-comm/sessiond-comm.h>
  23 #include <common/tracker.h>
  24 #include <common/utils.h>
  25 #include <lttng/event.h>
  26 #include <lttng/lttng-error.h>
  27 #include <lttng/tracker.h>
  28
  29 #include <lttng/userspace-probe.h>
  30 #include <lttng/userspace-probe-internal.h>
  31 #include <lttng/condition/event-rule-matches.h>
  32 #include <lttng/condition/event-rule-matches-internal.h>
  33 #include <lttng/event-rule/event-rule.h>
  34 #include <lttng/event-rule/event-rule-internal.h>
  35 #include <lttng/event-rule/kernel-uprobe-internal.h>
  36
  37 #include "event-notifier-error-accounting.h"
  38 #include "lttng-sessiond.h"
  39 #include "lttng-syscall.h"
  40 #include "condition-internal.h"
  41 #include "consumer.h"
  42 #include "kernel.h"
  43 #include "kernel-consumer.h"
  44 #include "kern-modules.h"
  45 #include "sessiond-config.h"
  46 #include "utils.h"
  47 #include "rotate.h"
  48 #include "modprobe.h"
  49 #include "tracker.h"
  50 #include "notification-thread-commands.h"
  51
  52 /*
  53  * Key used to reference a channel between the sessiond and the consumer. This
  54  * is only read and updated with the session_list lock held.
  55  */
  56 static uint64_t next_kernel_channel_key;
  57
  58 static const char *module_proc_lttng = "/proc/lttng";
  59
  60 static int kernel_tracer_fd = -1;
  61 static int kernel_tracer_event_notifier_group_fd = -1;
  62 static int kernel_tracer_event_notifier_group_notification_fd = -1;
  63 static struct cds_lfht *kernel_token_to_event_notifier_rule_ht;
  64
  65 /*
  66  * On some architectures, calling convention details are embedded in the symbol
  67  * addresses. Uprobe requires a "clean" symbol offset (or at least, an address
  68  * where an instruction boundary would be legal) to add
  69  * instrumentation. sanitize_uprobe_offset implements that sanitization logic on
  70  * a per-architecture basis.
  71  */
  72 #if defined(__arm__) || defined(__aarch64__)
  73 static inline uint64_t sanitize_uprobe_offset(uint64_t raw_offset)
  74 {
  75         /*
  76          * The least significant bit is used when branching to switch to thumb
  77          * ISA. However, it's an invalid address for us; mask the least
  78          * significant bit.
  79          */
  80         return raw_offset &= ~0b1;
  81 }
  82 #else /* defined(__arm__) || defined(__aarch64__) */
  83 static inline uint64_t sanitize_uprobe_offset(uint64_t raw_offset)
  84 {
  85         return raw_offset;
  86 }
  87 #endif
  88
  89 /*
  90  * Add context on a kernel channel.
  91  *
  92  * Assumes the ownership of ctx.
  93  */
  94 int kernel_add_channel_context(struct ltt_kernel_channel *chan,
  95                 struct ltt_kernel_context *ctx)
  96 {
  97         int ret;
  98
  99         assert(chan);
 100         assert(ctx);
 101
 102         DBG("Adding context to channel %s", chan->channel->name);
 103         ret = kernctl_add_context(chan->fd, &ctx->ctx);
 104         if (ret < 0) {
 105                 switch (-ret) {
 106                 case ENOSYS:
 107                         /* Exists but not available for this kernel */
 108                         ret = LTTNG_ERR_KERN_CONTEXT_UNAVAILABLE;
 109                         goto error;
 110                 case EEXIST:
 111                         /* If EEXIST, we just ignore the error */
 112                         ret = 0;
 113                         goto end;
 114                 default:
 115                         PERROR("add context ioctl");
 116                         ret = LTTNG_ERR_KERN_CONTEXT_FAIL;
 117                         goto error;
 118                 }
 119         }
 120         ret = 0;
 121
 122 end:
 123         cds_list_add_tail(&ctx->list, &chan->ctx_list);
 124         ctx->in_list = true;
 125         ctx = NULL;
 126 error:
 127         if (ctx) {
 128                 trace_kernel_destroy_context(ctx);
 129         }
 130         return ret;
 131 }
 132
 133 /*
 134  * Create a new kernel session, register it to the kernel tracer and add it to
 135  * the session daemon session.
 136  */
 137 int kernel_create_session(struct ltt_session *session)
 138 {
 139         int ret;
 140         struct ltt_kernel_session *lks;
 141
 142         assert(session);
 143
 144         /* Allocate data structure */
 145         lks = trace_kernel_create_session();
 146         if (lks == NULL) {
 147                 ret = -1;
 148                 goto error;
 149         }
 150
 151         /* Kernel tracer session creation */
 152         ret = kernctl_create_session(kernel_tracer_fd);
 153         if (ret < 0) {
 154                 PERROR("ioctl kernel create session");
 155                 goto error;
 156         }
 157
 158         lks->fd = ret;
 159         /* Prevent fd duplication after execlp() */
 160         ret = fcntl(lks->fd, F_SETFD, FD_CLOEXEC);
 161         if (ret < 0) {
 162                 PERROR("fcntl session fd");
 163         }
 164
 165         lks->id = session->id;
 166         lks->consumer_fds_sent = 0;
 167         session->kernel_session = lks;
 168
 169         DBG("Kernel session created (fd: %d)", lks->fd);
 170
 171         /*
 172          * This is necessary since the creation time is present in the session
 173          * name when it is generated.
 174          */
 175         if (session->has_auto_generated_name) {
 176                 ret = kernctl_session_set_name(lks->fd, DEFAULT_SESSION_NAME);
 177         } else {
 178                 ret = kernctl_session_set_name(lks->fd, session->name);
 179         }
 180         if (ret) {
 181                 WARN("Could not set kernel session name for session %" PRIu64 " name: %s",
 182                         session->id, session->name);
 183         }
 184
 185         ret = kernctl_session_set_creation_time(lks->fd, session->creation_time);
 186         if (ret) {
 187                 WARN("Could not set kernel session creation time for session %" PRIu64 " name: %s",
 188                         session->id, session->name);
 189         }
 190
 191         return 0;
 192
 193 error:
 194         if (lks) {
 195                 trace_kernel_destroy_session(lks);
 196                 trace_kernel_free_session(lks);
 197         }
 198         return ret;
 199 }
 200
 201 /*
 202  * Create a kernel channel, register it to the kernel tracer and add it to the
 203  * kernel session.
 204  */
 205 int kernel_create_channel(struct ltt_kernel_session *session,
 206                 struct lttng_channel *chan)
 207 {
 208         int ret;
 209         struct ltt_kernel_channel *lkc;
 210
 211         assert(session);
 212         assert(chan);
 213
 214         /* Allocate kernel channel */
 215         lkc = trace_kernel_create_channel(chan);
 216         if (lkc == NULL) {
 217                 goto error;
 218         }
 219
 220         DBG3("Kernel create channel %s with attr: %d, %" PRIu64 ", %" PRIu64 ", %u, %u, %d, %d",
 221                         chan->name, lkc->channel->attr.overwrite,
 222                         lkc->channel->attr.subbuf_size, lkc->channel->attr.num_subbuf,
 223                         lkc->channel->attr.switch_timer_interval, lkc->channel->attr.read_timer_interval,
 224                         lkc->channel->attr.live_timer_interval, lkc->channel->attr.output);
 225
 226         /* Kernel tracer channel creation */
 227         ret = kernctl_create_channel(session->fd, &lkc->channel->attr);
 228         if (ret < 0) {
 229                 PERROR("ioctl kernel create channel");
 230                 goto error;
 231         }
 232
 233         /* Setup the channel fd */
 234         lkc->fd = ret;
 235         /* Prevent fd duplication after execlp() */
 236         ret = fcntl(lkc->fd, F_SETFD, FD_CLOEXEC);
 237         if (ret < 0) {
 238                 PERROR("fcntl session fd");
 239         }
 240
 241         /* Add channel to session */
 242         cds_list_add(&lkc->list, &session->channel_list.head);
 243         session->channel_count++;
 244         lkc->session = session;
 245         lkc->key = ++next_kernel_channel_key;
 246
 247         DBG("Kernel channel %s created (fd: %d, key: %" PRIu64 ")",
 248                         lkc->channel->name, lkc->fd, lkc->key);
 249
 250         return 0;
 251
 252 error:
 253         if (lkc) {
 254                 free(lkc->channel);
 255                 free(lkc);
 256         }
 257         return -1;
 258 }
 259
 260 /*
 261  * Create a kernel event notifier group, register it to the kernel tracer and
 262  * add it to the kernel session.
 263  */
 264 static int kernel_create_event_notifier_group(int *event_notifier_group_fd)
 265 {
 266         int ret;
 267         int local_fd = -1;
 268
 269         assert(event_notifier_group_fd);
 270
 271         /* Kernel event notifier group creation. */
 272         ret = kernctl_create_event_notifier_group(kernel_tracer_fd);
 273         if (ret < 0) {
 274                 PERROR("Failed to create kernel event notifier group");
 275                 ret = -1;
 276                 goto error;
 277         }
 278
 279         local_fd = ret;
 280
 281         /* Prevent fd duplication after execlp(). */
 282         ret = fcntl(local_fd, F_SETFD, FD_CLOEXEC);
 283         if (ret < 0) {
 284                 PERROR("Failed to set FD_CLOEXEC on kernel event notifier group file descriptor: fd = %d",
 285                                 local_fd);
 286                 goto error;
 287         }
 288
 289         DBG("Created kernel event notifier group: fd = %d", local_fd);
 290         *event_notifier_group_fd = local_fd;
 291         local_fd = -1;
 292         ret = 0;
 293 error:
 294         if (local_fd >= 0) {
 295                 ret = close(local_fd);
 296                 if (ret) {
 297                         PERROR("Failed to close kernel event notifier group file descriptor: fd = %d",
 298                                         local_fd);
 299                 }
 300         }
 301
 302         return ret;
 303 }
 304
 305 /*
 306  * Compute the offset of the instrumentation byte in the binary based on the
 307  * function probe location using the ELF lookup method.
 308  *
 309  * Returns 0 on success and set the offset out parameter to the offset of the
 310  * elf symbol
 311  * Returns -1 on error
 312  */
 313 static
 314 int extract_userspace_probe_offset_function_elf(
 315                 const struct lttng_userspace_probe_location *probe_location,
 316                 uid_t uid, gid_t gid, uint64_t *offset)
 317 {
 318         int fd;
 319         int ret = 0;
 320         const char *symbol = NULL;
 321         const struct lttng_userspace_probe_location_lookup_method *lookup = NULL;
 322         enum lttng_userspace_probe_location_lookup_method_type lookup_method_type;
 323
 324         assert(lttng_userspace_probe_location_get_type(probe_location) ==
 325                         LTTNG_USERSPACE_PROBE_LOCATION_TYPE_FUNCTION);
 326
 327         lookup = lttng_userspace_probe_location_get_lookup_method(
 328                         probe_location);
 329         if (!lookup) {
 330                 ret = -1;
 331                 goto end;
 332         }
 333
 334         lookup_method_type =
 335                         lttng_userspace_probe_location_lookup_method_get_type(lookup);
 336
 337         assert(lookup_method_type ==
 338                         LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_FUNCTION_ELF);
 339
 340         symbol = lttng_userspace_probe_location_function_get_function_name(
 341                         probe_location);
 342         if (!symbol) {
 343                 ret = -1;
 344                 goto end;
 345         }
 346
 347         fd = lttng_userspace_probe_location_function_get_binary_fd(probe_location);
 348         if (fd < 0) {
 349                 ret = -1;
 350                 goto end;
 351         }
 352
 353         ret = run_as_extract_elf_symbol_offset(fd, symbol, uid, gid, offset);
 354         if (ret < 0) {
 355                 DBG("userspace probe offset calculation failed for "
 356                                 "function %s", symbol);
 357                 goto end;
 358         }
 359
 360         DBG("userspace probe elf offset for %s is 0x%jd", symbol, (intmax_t)(*offset));
 361 end:
 362         return ret;
 363 }
 364
 365 /*
 366  * Compute the offsets of the instrumentation bytes in the binary based on the
 367  * tracepoint probe location using the SDT lookup method. This function
 368  * allocates the offsets buffer, the caller must free it.
 369  *
 370  * Returns 0 on success and set the offset out parameter to the offsets of the
 371  * SDT tracepoint.
 372  * Returns -1 on error.
 373  */
 374 static
 375 int extract_userspace_probe_offset_tracepoint_sdt(
 376                 const struct lttng_userspace_probe_location *probe_location,
 377                 uid_t uid, gid_t gid, uint64_t **offsets,
 378                 uint32_t *offsets_count)
 379 {
 380         enum lttng_userspace_probe_location_lookup_method_type lookup_method_type;
 381         const struct lttng_userspace_probe_location_lookup_method *lookup = NULL;
 382         const char *probe_name = NULL, *provider_name = NULL;
 383         int ret = 0;
 384         int fd, i;
 385
 386         assert(lttng_userspace_probe_location_get_type(probe_location) ==
 387                         LTTNG_USERSPACE_PROBE_LOCATION_TYPE_TRACEPOINT);
 388
 389         lookup = lttng_userspace_probe_location_get_lookup_method(probe_location);
 390         if (!lookup) {
 391                 ret = -1;
 392                 goto end;
 393         }
 394
 395         lookup_method_type =
 396                         lttng_userspace_probe_location_lookup_method_get_type(lookup);
 397
 398         assert(lookup_method_type ==
 399                         LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_TRACEPOINT_SDT);
 400
 401
 402         probe_name = lttng_userspace_probe_location_tracepoint_get_probe_name(
 403                         probe_location);
 404         if (!probe_name) {
 405                 ret = -1;
 406                 goto end;
 407         }
 408
 409         provider_name = lttng_userspace_probe_location_tracepoint_get_provider_name(
 410                         probe_location);
 411         if (!provider_name) {
 412                 ret = -1;
 413                 goto end;
 414         }
 415
 416         fd = lttng_userspace_probe_location_tracepoint_get_binary_fd(probe_location);
 417         if (fd < 0) {
 418                 ret = -1;
 419                 goto end;
 420         }
 421
 422         ret = run_as_extract_sdt_probe_offsets(fd, provider_name, probe_name,
 423                         uid, gid, offsets, offsets_count);
 424         if (ret < 0) {
 425                 DBG("userspace probe offset calculation failed for sdt "
 426                                 "probe %s:%s", provider_name, probe_name);
 427                 goto end;
 428         }
 429
 430         if (*offsets_count == 0) {
 431                 DBG("no userspace probe offset found");
 432                 goto end;
 433         }
 434
 435         DBG("%u userspace probe SDT offsets found for %s:%s at:",
 436                         *offsets_count, provider_name, probe_name);
 437         for (i = 0; i < *offsets_count; i++) {
 438                 DBG("\t0x%jd", (intmax_t)((*offsets)[i]));
 439         }
 440 end:
 441         return ret;
 442 }
 443
 444 static
 445 int userspace_probe_add_callsite(
 446                 const struct lttng_userspace_probe_location *location,
 447                 uid_t uid, gid_t gid, int fd)
 448 {
 449         const struct lttng_userspace_probe_location_lookup_method *lookup_method = NULL;
 450         enum lttng_userspace_probe_location_lookup_method_type type;
 451         int ret;
 452
 453         lookup_method = lttng_userspace_probe_location_get_lookup_method(location);
 454         if (!lookup_method) {
 455                 ret = -1;
 456                 goto end;
 457         }
 458
 459         type = lttng_userspace_probe_location_lookup_method_get_type(lookup_method);
 460         switch (type) {
 461         case LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_FUNCTION_ELF:
 462         {
 463                 struct lttng_kernel_abi_event_callsite callsite;
 464                 uint64_t offset;
 465
 466                 ret = extract_userspace_probe_offset_function_elf(location,
 467                                 uid, gid, &offset);
 468                 if (ret) {
 469                         ret = LTTNG_ERR_PROBE_LOCATION_INVAL;
 470                         goto end;
 471                 }
 472
 473                 callsite.u.uprobe.offset = sanitize_uprobe_offset(offset);
 474                 ret = kernctl_add_callsite(fd, &callsite);
 475                 if (ret) {
 476                         WARN("Failed to add callsite to ELF userspace probe.");
 477                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 478                         goto end;
 479                 }
 480                 break;
 481         }
 482         case LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_TRACEPOINT_SDT:
 483         {
 484                 int i;
 485                 uint64_t *offsets = NULL;
 486                 uint32_t offsets_count;
 487                 struct lttng_kernel_abi_event_callsite callsite;
 488
 489                 /*
 490                  * This call allocates the offsets buffer. This buffer must be freed
 491                  * by the caller
 492                  */
 493                 ret = extract_userspace_probe_offset_tracepoint_sdt(location,
 494                                 uid, gid, &offsets, &offsets_count);
 495                 if (ret) {
 496                         ret = LTTNG_ERR_PROBE_LOCATION_INVAL;
 497                         goto end;
 498                 }
 499                 for (i = 0; i < offsets_count; i++) {
 500                         callsite.u.uprobe.offset = sanitize_uprobe_offset(offsets[i]);
 501                         ret = kernctl_add_callsite(fd, &callsite);
 502                         if (ret) {
 503                                 WARN("Failed to add callsite to SDT userspace probe");
 504                                 ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 505                                 free(offsets);
 506                                 goto end;
 507                         }
 508                 }
 509                 free(offsets);
 510                 break;
 511         }
 512         default:
 513                 ret = LTTNG_ERR_PROBE_LOCATION_INVAL;
 514                 goto end;
 515         }
 516 end:
 517         return ret;
 518 }
 519
 520 /*
 521  * Extract the offsets of the instrumentation point for the different lookup
 522  * methods.
 523  */
 524 static
 525 int userspace_probe_event_add_callsites(struct lttng_event *ev,
 526                         struct ltt_kernel_session *session, int fd)
 527 {
 528         int ret;
 529         const struct lttng_userspace_probe_location *location = NULL;
 530
 531         assert(ev);
 532         assert(ev->type == LTTNG_EVENT_USERSPACE_PROBE);
 533
 534         location = lttng_event_get_userspace_probe_location(ev);
 535         if (!location) {
 536                 ret = -1;
 537                 goto end;
 538         }
 539
 540         ret = userspace_probe_add_callsite(location, session->uid, session->gid,
 541                 fd);
 542         if (ret) {
 543                 WARN("Failed to add callsite to userspace probe event '%s'",
 544                                 ev->name);
 545         }
 546
 547 end:
 548         return ret;
 549 }
 550
 551 /*
 552  * Extract the offsets of the instrumentation point for the different look-up
 553  * methods.
 554  */
 555 static int userspace_probe_event_rule_add_callsites(
 556                 const struct lttng_event_rule *rule,
 557                 const struct lttng_credentials *creds,
 558                 int fd)
 559 {
 560         int ret;
 561         enum lttng_event_rule_status status;
 562         enum lttng_event_rule_type event_rule_type;
 563         const struct lttng_userspace_probe_location *location = NULL;
 564
 565         assert(rule);
 566         assert(creds);
 567
 568         event_rule_type = lttng_event_rule_get_type(rule);
 569         assert(event_rule_type == LTTNG_EVENT_RULE_TYPE_KERNEL_UPROBE);
 570
 571         status = lttng_event_rule_kernel_uprobe_get_location(rule, &location);
 572         if (status != LTTNG_EVENT_RULE_STATUS_OK || !location) {
 573                 ret = -1;
 574                 goto end;
 575         }
 576
 577         ret = userspace_probe_add_callsite(location,
 578                         lttng_credentials_get_uid(creds),
 579                         lttng_credentials_get_gid(creds), fd);
 580         if (ret) {
 581                 WARN("Failed to add callsite to user space probe object: fd = %d",
 582                                 fd);
 583         }
 584
 585 end:
 586         return ret;
 587 }
 588
 589 /*
 590  * Create a kernel event, enable it to the kernel tracer and add it to the
 591  * channel event list of the kernel session.
 592  * We own filter_expression and filter.
 593  */
 594 int kernel_create_event(struct lttng_event *ev,
 595                 struct ltt_kernel_channel *channel,
 596                 char *filter_expression,
 597                 struct lttng_bytecode *filter)
 598 {
 599         int err, fd;
 600         enum lttng_error_code ret;
 601         struct ltt_kernel_event *event;
 602
 603         assert(ev);
 604         assert(channel);
 605
 606         /* We pass ownership of filter_expression and filter */
 607         ret = trace_kernel_create_event(ev, filter_expression,
 608                         filter, &event);
 609         if (ret != LTTNG_OK) {
 610                 goto error;
 611         }
 612
 613         fd = kernctl_create_event(channel->fd, event->event);
 614         if (fd < 0) {
 615                 switch (-fd) {
 616                 case EEXIST:
 617                         ret = LTTNG_ERR_KERN_EVENT_EXIST;
 618                         break;
 619                 case ENOSYS:
 620                         WARN("Event type not implemented");
 621                         ret = LTTNG_ERR_KERN_EVENT_ENOSYS;
 622                         break;
 623                 case ENOENT:
 624                         WARN("Event %s not found!", ev->name);
 625                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 626                         break;
 627                 default:
 628                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 629                         PERROR("create event ioctl");
 630                 }
 631                 goto free_event;
 632         }
 633
 634         event->type = ev->type;
 635         event->fd = fd;
 636         /* Prevent fd duplication after execlp() */
 637         err = fcntl(event->fd, F_SETFD, FD_CLOEXEC);
 638         if (err < 0) {
 639                 PERROR("fcntl session fd");
 640         }
 641
 642         if (filter) {
 643                 err = kernctl_filter(event->fd, filter);
 644                 if (err < 0) {
 645                         switch (-err) {
 646                         case ENOMEM:
 647                                 ret = LTTNG_ERR_FILTER_NOMEM;
 648                                 break;
 649                         default:
 650                                 ret = LTTNG_ERR_FILTER_INVAL;
 651                                 break;
 652                         }
 653                         goto filter_error;
 654                 }
 655         }
 656
 657         if (ev->type == LTTNG_EVENT_USERSPACE_PROBE) {
 658                 ret = userspace_probe_event_add_callsites(ev, channel->session,
 659                                 event->fd);
 660                 if (ret) {
 661                         goto add_callsite_error;
 662                 }
 663         }
 664
 665         err = kernctl_enable(event->fd);
 666         if (err < 0) {
 667                 switch (-err) {
 668                 case EEXIST:
 669                         ret = LTTNG_ERR_KERN_EVENT_EXIST;
 670                         break;
 671                 default:
 672                         PERROR("enable kernel event");
 673                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 674                         break;
 675                 }
 676                 goto enable_error;
 677         }
 678
 679         /* Add event to event list */
 680         cds_list_add(&event->list, &channel->events_list.head);
 681         channel->event_count++;
 682
 683         DBG("Event %s created (fd: %d)", ev->name, event->fd);
 684
 685         return 0;
 686
 687 add_callsite_error:
 688 enable_error:
 689 filter_error:
 690         {
 691                 int closeret;
 692
 693                 closeret = close(event->fd);
 694                 if (closeret) {
 695                         PERROR("close event fd");
 696                 }
 697         }
 698 free_event:
 699         free(event);
 700 error:
 701         return ret;
 702 }
 703
 704 /*
 705  * Disable a kernel channel.
 706  */
 707 int kernel_disable_channel(struct ltt_kernel_channel *chan)
 708 {
 709         int ret;
 710
 711         assert(chan);
 712
 713         ret = kernctl_disable(chan->fd);
 714         if (ret < 0) {
 715                 PERROR("disable chan ioctl");
 716                 goto error;
 717         }
 718
 719         chan->enabled = 0;
 720         DBG("Kernel channel %s disabled (fd: %d, key: %" PRIu64 ")",
 721                         chan->channel->name, chan->fd, chan->key);
 722
 723         return 0;
 724
 725 error:
 726         return ret;
 727 }
 728
 729 /*
 730  * Enable a kernel channel.
 731  */
 732 int kernel_enable_channel(struct ltt_kernel_channel *chan)
 733 {
 734         int ret;
 735
 736         assert(chan);
 737
 738         ret = kernctl_enable(chan->fd);
 739         if (ret < 0 && ret != -EEXIST) {
 740                 PERROR("Enable kernel chan");
 741                 goto error;
 742         }
 743
 744         chan->enabled = 1;
 745         DBG("Kernel channel %s enabled (fd: %d, key: %" PRIu64 ")",
 746                         chan->channel->name, chan->fd, chan->key);
 747
 748         return 0;
 749
 750 error:
 751         return ret;
 752 }
 753
 754 /*
 755  * Enable a kernel event.
 756  */
 757 int kernel_enable_event(struct ltt_kernel_event *event)
 758 {
 759         int ret;
 760
 761         assert(event);
 762
 763         ret = kernctl_enable(event->fd);
 764         if (ret < 0) {
 765                 switch (-ret) {
 766                 case EEXIST:
 767                         ret = LTTNG_ERR_KERN_EVENT_EXIST;
 768                         break;
 769                 default:
 770                         PERROR("enable kernel event");
 771                         break;
 772                 }
 773                 goto error;
 774         }
 775
 776         event->enabled = 1;
 777         DBG("Kernel event %s enabled (fd: %d)", event->event->name, event->fd);
 778
 779         return 0;
 780
 781 error:
 782         return ret;
 783 }
 784
 785 /*
 786  * Disable a kernel event.
 787  */
 788 int kernel_disable_event(struct ltt_kernel_event *event)
 789 {
 790         int ret;
 791
 792         assert(event);
 793
 794         ret = kernctl_disable(event->fd);
 795         if (ret < 0) {
 796                 PERROR("Failed to disable kernel event: name = '%s', fd = %d",
 797                                 event->event->name, event->fd);
 798                 goto error;
 799         }
 800
 801         event->enabled = 0;
 802         DBG("Kernel event %s disabled (fd: %d)", event->event->name, event->fd);
 803
 804         return 0;
 805
 806 error:
 807         return ret;
 808 }
 809
 810 /*
 811  * Disable a kernel event notifier.
 812  */
 813 static
 814 int kernel_disable_event_notifier_rule(struct ltt_kernel_event_notifier_rule *event)
 815 {
 816         int ret;
 817
 818         assert(event);
 819
 820         rcu_read_lock();
 821         cds_lfht_del(kernel_token_to_event_notifier_rule_ht, &event->ht_node);
 822         rcu_read_unlock();
 823
 824         ret = kernctl_disable(event->fd);
 825         if (ret < 0) {
 826                 PERROR("Failed to disable kernel event notifier: fd = %d, token = %" PRIu64,
 827                                 event->fd, event->token);
 828                 goto error;
 829         }
 830
 831         event->enabled = 0;
 832         DBG("Disabled kernel event notifier: fd = %d, token = %" PRIu64,
 833                         event->fd, event->token);
 834
 835 error:
 836         return ret;
 837 }
 838
 839 static
 840 struct process_attr_tracker *_kernel_get_process_attr_tracker(
 841                 struct ltt_kernel_session *session,
 842                 enum lttng_process_attr process_attr)
 843 {
 844         switch (process_attr) {
 845         case LTTNG_PROCESS_ATTR_PROCESS_ID:
 846                 return session->tracker_pid;
 847         case LTTNG_PROCESS_ATTR_VIRTUAL_PROCESS_ID:
 848                 return session->tracker_vpid;
 849         case LTTNG_PROCESS_ATTR_USER_ID:
 850                 return session->tracker_uid;
 851         case LTTNG_PROCESS_ATTR_VIRTUAL_USER_ID:
 852                 return session->tracker_vuid;
 853         case LTTNG_PROCESS_ATTR_GROUP_ID:
 854                 return session->tracker_gid;
 855         case LTTNG_PROCESS_ATTR_VIRTUAL_GROUP_ID:
 856                 return session->tracker_vgid;
 857         default:
 858                 return NULL;
 859         }
 860 }
 861
 862 const struct process_attr_tracker *kernel_get_process_attr_tracker(
 863                 struct ltt_kernel_session *session,
 864                 enum lttng_process_attr process_attr)
 865 {
 866         return (const struct process_attr_tracker *)
 867                         _kernel_get_process_attr_tracker(session, process_attr);
 868 }
 869
 870 enum lttng_error_code kernel_process_attr_tracker_set_tracking_policy(
 871                 struct ltt_kernel_session *session,
 872                 enum lttng_process_attr process_attr,
 873                 enum lttng_tracking_policy policy)
 874 {
 875         int ret;
 876         enum lttng_error_code ret_code = LTTNG_OK;
 877         struct process_attr_tracker *tracker =
 878                         _kernel_get_process_attr_tracker(session, process_attr);
 879         enum lttng_tracking_policy previous_policy;
 880
 881         if (!tracker) {
 882                 ret_code = LTTNG_ERR_INVALID;
 883                 goto end;
 884         }
 885
 886         previous_policy = process_attr_tracker_get_tracking_policy(tracker);
 887         ret = process_attr_tracker_set_tracking_policy(tracker, policy);
 888         if (ret) {
 889                 ret_code = LTTNG_ERR_UNK;
 890                 goto end;
 891         }
 892
 893         if (previous_policy == policy) {
 894                 goto end;
 895         }
 896
 897         switch (policy) {
 898         case LTTNG_TRACKING_POLICY_INCLUDE_ALL:
 899                 if (process_attr == LTTNG_PROCESS_ATTR_PROCESS_ID) {
 900                         /*
 901                          * Maintain a special case for the process ID process
 902                          * attribute tracker as it was the only supported
 903                          * attribute prior to 2.12.
 904                          */
 905                         ret = kernctl_track_pid(session->fd, -1);
 906                 } else {
 907                         ret = kernctl_track_id(session->fd, process_attr, -1);
 908                 }
 909                 break;
 910         case LTTNG_TRACKING_POLICY_EXCLUDE_ALL:
 911         case LTTNG_TRACKING_POLICY_INCLUDE_SET:
 912                 /* fall-through. */
 913                 if (process_attr == LTTNG_PROCESS_ATTR_PROCESS_ID) {
 914                         /*
 915                          * Maintain a special case for the process ID process
 916                          * attribute tracker as it was the only supported
 917                          * attribute prior to 2.12.
 918                          */
 919                         ret = kernctl_untrack_pid(session->fd, -1);
 920                 } else {
 921                         ret = kernctl_untrack_id(session->fd, process_attr, -1);
 922                 }
 923                 break;
 924         default:
 925                 abort();
 926         }
 927         /* kern-ctl error handling */
 928         switch (-ret) {
 929         case 0:
 930                 ret_code = LTTNG_OK;
 931                 break;
 932         case EINVAL:
 933                 ret_code = LTTNG_ERR_INVALID;
 934                 break;
 935         case ENOMEM:
 936                 ret_code = LTTNG_ERR_NOMEM;
 937                 break;
 938         case EEXIST:
 939                 ret_code = LTTNG_ERR_PROCESS_ATTR_EXISTS;
 940                 break;
 941         default:
 942                 ret_code = LTTNG_ERR_UNK;
 943                 break;
 944         }
 945 end:
 946         return ret_code;
 947 }
 948
 949 enum lttng_error_code kernel_process_attr_tracker_inclusion_set_add_value(
 950                 struct ltt_kernel_session *session,
 951                 enum lttng_process_attr process_attr,
 952                 const struct process_attr_value *value)
 953 {
 954         int ret, integral_value;
 955         enum lttng_error_code ret_code;
 956         struct process_attr_tracker *tracker;
 957         enum process_attr_tracker_status status;
 958
 959         /*
 960          * Convert process attribute tracker value to the integral
 961          * representation required by the kern-ctl API.
 962          */
 963         switch (process_attr) {
 964         case LTTNG_PROCESS_ATTR_PROCESS_ID:
 965         case LTTNG_PROCESS_ATTR_VIRTUAL_PROCESS_ID:
 966                 integral_value = (int) value->value.pid;
 967                 break;
 968         case LTTNG_PROCESS_ATTR_USER_ID:
 969         case LTTNG_PROCESS_ATTR_VIRTUAL_USER_ID:
 970                 if (value->type == LTTNG_PROCESS_ATTR_VALUE_TYPE_USER_NAME) {
 971                         uid_t uid;
 972
 973                         ret_code = utils_user_id_from_name(
 974                                         value->value.user_name, &uid);
 975                         if (ret_code != LTTNG_OK) {
 976                                 goto end;
 977                         }
 978                         integral_value = (int) uid;
 979                 } else {
 980                         integral_value = (int) value->value.uid;
 981                 }
 982                 break;
 983         case LTTNG_PROCESS_ATTR_GROUP_ID:
 984         case LTTNG_PROCESS_ATTR_VIRTUAL_GROUP_ID:
 985                 if (value->type == LTTNG_PROCESS_ATTR_VALUE_TYPE_GROUP_NAME) {
 986                         gid_t gid;
 987
 988                         ret_code = utils_group_id_from_name(
 989                                         value->value.group_name, &gid);
 990                         if (ret_code != LTTNG_OK) {
 991                                 goto end;
 992                         }
 993                         integral_value = (int) gid;
 994                 } else {
 995                         integral_value = (int) value->value.gid;
 996                 }
 997                 break;
 998         default:
 999                 ret_code = LTTNG_ERR_INVALID;
1000                 goto end;
1001         }
1002
1003         tracker = _kernel_get_process_attr_tracker(session, process_attr);
1004         if (!tracker) {
1005                 ret_code = LTTNG_ERR_INVALID;
1006                 goto end;
1007         }
1008
1009         status = process_attr_tracker_inclusion_set_add_value(tracker, value);
1010         if (status != PROCESS_ATTR_TRACKER_STATUS_OK) {
1011                 switch (status) {
1012                 case PROCESS_ATTR_TRACKER_STATUS_EXISTS:
1013                         ret_code = LTTNG_ERR_PROCESS_ATTR_EXISTS;
1014                         break;
1015                 case PROCESS_ATTR_TRACKER_STATUS_INVALID_TRACKING_POLICY:
1016                         ret_code = LTTNG_ERR_PROCESS_ATTR_TRACKER_INVALID_TRACKING_POLICY;
1017                         break;
1018                 case PROCESS_ATTR_TRACKER_STATUS_ERROR:
1019                 default:
1020                         ret_code = LTTNG_ERR_UNK;
1021                         break;
1022                 }
1023                 goto end;
1024         }
1025
1026         DBG("Kernel track %s %d for session id %" PRIu64,
1027                         lttng_process_attr_to_string(process_attr),
1028                         integral_value, session->id);
1029         if (process_attr == LTTNG_PROCESS_ATTR_PROCESS_ID) {
1030                 /*
1031                  * Maintain a special case for the process ID process attribute
1032                  * tracker as it was the only supported attribute prior to 2.12.
1033                  */
1034                 ret = kernctl_track_pid(session->fd, integral_value);
1035         } else {
1036                 ret = kernctl_track_id(
1037                                 session->fd, process_attr, integral_value);
1038         }
1039         if (ret == 0) {
1040                 ret_code = LTTNG_OK;
1041                 goto end;
1042         }
1043
1044         kernel_wait_quiescent();
1045
1046         /* kern-ctl error handling */
1047         switch (-ret) {
1048         case 0:
1049                 ret_code = LTTNG_OK;
1050                 break;
1051         case EINVAL:
1052                 ret_code = LTTNG_ERR_INVALID;
1053                 break;
1054         case ENOMEM:
1055                 ret_code = LTTNG_ERR_NOMEM;
1056                 break;
1057         case EEXIST:
1058                 ret_code = LTTNG_ERR_PROCESS_ATTR_EXISTS;
1059                 break;
1060         default:
1061                 ret_code = LTTNG_ERR_UNK;
1062                 break;
1063         }
1064
1065         /* Attempt to remove the value from the tracker. */
1066         status = process_attr_tracker_inclusion_set_remove_value(
1067                         tracker, value);
1068         if (status != PROCESS_ATTR_TRACKER_STATUS_OK) {
1069                 ERR("Failed to roll-back the tracking of kernel %s process attribute %d while handling a kern-ctl error",
1070                                 lttng_process_attr_to_string(process_attr),
1071                                 integral_value);
1072         }
1073 end:
1074         return ret_code;
1075 }
1076
1077 enum lttng_error_code kernel_process_attr_tracker_inclusion_set_remove_value(
1078                 struct ltt_kernel_session *session,
1079                 enum lttng_process_attr process_attr,
1080                 const struct process_attr_value *value)
1081 {
1082         int ret, integral_value;
1083         enum lttng_error_code ret_code;
1084         struct process_attr_tracker *tracker;
1085         enum process_attr_tracker_status status;
1086
1087         /*
1088          * Convert process attribute tracker value to the integral
1089          * representation required by the kern-ctl API.
1090          */
1091         switch (process_attr) {
1092         case LTTNG_PROCESS_ATTR_PROCESS_ID:
1093         case LTTNG_PROCESS_ATTR_VIRTUAL_PROCESS_ID:
1094                 integral_value = (int) value->value.pid;
1095                 break;
1096         case LTTNG_PROCESS_ATTR_USER_ID:
1097         case LTTNG_PROCESS_ATTR_VIRTUAL_USER_ID:
1098                 if (value->type == LTTNG_PROCESS_ATTR_VALUE_TYPE_USER_NAME) {
1099                         uid_t uid;
1100
1101                         ret_code = utils_user_id_from_name(
1102                                         value->value.user_name, &uid);
1103                         if (ret_code != LTTNG_OK) {
1104                                 goto end;
1105                         }
1106                         integral_value = (int) uid;
1107                 } else {
1108                         integral_value = (int) value->value.uid;
1109                 }
1110                 break;
1111         case LTTNG_PROCESS_ATTR_GROUP_ID:
1112         case LTTNG_PROCESS_ATTR_VIRTUAL_GROUP_ID:
1113                 if (value->type == LTTNG_PROCESS_ATTR_VALUE_TYPE_GROUP_NAME) {
1114                         gid_t gid;
1115
1116                         ret_code = utils_group_id_from_name(
1117                                         value->value.group_name, &gid);
1118                         if (ret_code != LTTNG_OK) {
1119                                 goto end;
1120                         }
1121                         integral_value = (int) gid;
1122                 } else {
1123                         integral_value = (int) value->value.gid;
1124                 }
1125                 break;
1126         default:
1127                 ret_code = LTTNG_ERR_INVALID;
1128                 goto end;
1129         }
1130
1131         tracker = _kernel_get_process_attr_tracker(session, process_attr);
1132         if (!tracker) {
1133                 ret_code = LTTNG_ERR_INVALID;
1134                 goto end;
1135         }
1136
1137         status = process_attr_tracker_inclusion_set_remove_value(
1138                         tracker, value);
1139         if (status != PROCESS_ATTR_TRACKER_STATUS_OK) {
1140                 switch (status) {
1141                 case PROCESS_ATTR_TRACKER_STATUS_MISSING:
1142                         ret_code = LTTNG_ERR_PROCESS_ATTR_MISSING;
1143                         break;
1144                 case PROCESS_ATTR_TRACKER_STATUS_INVALID_TRACKING_POLICY:
1145                         ret_code = LTTNG_ERR_PROCESS_ATTR_TRACKER_INVALID_TRACKING_POLICY;
1146                         break;
1147                 case PROCESS_ATTR_TRACKER_STATUS_ERROR:
1148                 default:
1149                         ret_code = LTTNG_ERR_UNK;
1150                         break;
1151                 }
1152                 goto end;
1153         }
1154
1155         DBG("Kernel track %s %d for session id %" PRIu64,
1156                         lttng_process_attr_to_string(process_attr),
1157                         integral_value, session->id);
1158         if (process_attr == LTTNG_PROCESS_ATTR_PROCESS_ID) {
1159                 /*
1160                  * Maintain a special case for the process ID process attribute
1161                  * tracker as it was the only supported attribute prior to 2.12.
1162                  */
1163                 ret = kernctl_untrack_pid(session->fd, integral_value);
1164         } else {
1165                 ret = kernctl_untrack_id(
1166                                 session->fd, process_attr, integral_value);
1167         }
1168         if (ret == 0) {
1169                 ret_code = LTTNG_OK;
1170                 goto end;
1171         }
1172         kernel_wait_quiescent();
1173
1174         /* kern-ctl error handling */
1175         switch (-ret) {
1176         case 0:
1177                 ret_code = LTTNG_OK;
1178                 break;
1179         case EINVAL:
1180                 ret_code = LTTNG_ERR_INVALID;
1181                 break;
1182         case ENOMEM:
1183                 ret_code = LTTNG_ERR_NOMEM;
1184                 break;
1185         case ENOENT:
1186                 ret_code = LTTNG_ERR_PROCESS_ATTR_MISSING;
1187                 break;
1188         default:
1189                 ret_code = LTTNG_ERR_UNK;
1190                 break;
1191         }
1192
1193         /* Attempt to add the value to the tracker. */
1194         status = process_attr_tracker_inclusion_set_add_value(
1195                         tracker, value);
1196         if (status != PROCESS_ATTR_TRACKER_STATUS_OK) {
1197                 ERR("Failed to roll-back the tracking of kernel %s process attribute %d while handling a kern-ctl error",
1198                                 lttng_process_attr_to_string(process_attr),
1199                                 integral_value);
1200         }
1201 end:
1202         return ret_code;
1203 }
1204
1205 /*
1206  * Create kernel metadata, open from the kernel tracer and add it to the
1207  * kernel session.
1208  */
1209 int kernel_open_metadata(struct ltt_kernel_session *session)
1210 {
1211         int ret;
1212         struct ltt_kernel_metadata *lkm = NULL;
1213
1214         assert(session);
1215
1216         /* Allocate kernel metadata */
1217         lkm = trace_kernel_create_metadata();
1218         if (lkm == NULL) {
1219                 goto error;
1220         }
1221
1222         /* Kernel tracer metadata creation */
1223         ret = kernctl_open_metadata(session->fd, &lkm->conf->attr);
1224         if (ret < 0) {
1225                 goto error_open;
1226         }
1227
1228         lkm->fd = ret;
1229         lkm->key = ++next_kernel_channel_key;
1230         /* Prevent fd duplication after execlp() */
1231         ret = fcntl(lkm->fd, F_SETFD, FD_CLOEXEC);
1232         if (ret < 0) {
1233                 PERROR("fcntl session fd");
1234         }
1235
1236         session->metadata = lkm;
1237
1238         DBG("Kernel metadata opened (fd: %d)", lkm->fd);
1239
1240         return 0;
1241
1242 error_open:
1243         trace_kernel_destroy_metadata(lkm);
1244 error:
1245         return -1;
1246 }
1247
1248 /*
1249  * Start tracing session.
1250  */
1251 int kernel_start_session(struct ltt_kernel_session *session)
1252 {
1253         int ret;
1254
1255         assert(session);
1256
1257         ret = kernctl_start_session(session->fd);
1258         if (ret < 0) {
1259                 PERROR("ioctl start session");
1260                 goto error;
1261         }
1262
1263         DBG("Kernel session started");
1264
1265         return 0;
1266
1267 error:
1268         return ret;
1269 }
1270
1271 /*
1272  * Make a kernel wait to make sure in-flight probe have completed.
1273  */
1274 void kernel_wait_quiescent(void)
1275 {
1276         int ret;
1277         int fd = kernel_tracer_fd;
1278
1279         DBG("Kernel quiescent wait on %d", fd);
1280
1281         ret = kernctl_wait_quiescent(fd);
1282         if (ret < 0) {
1283                 PERROR("wait quiescent ioctl");
1284                 ERR("Kernel quiescent wait failed");
1285         }
1286 }
1287
1288 /*
1289  *  Force flush buffer of metadata.
1290  */
1291 int kernel_metadata_flush_buffer(int fd)
1292 {
1293         int ret;
1294
1295         DBG("Kernel flushing metadata buffer on fd %d", fd);
1296
1297         ret = kernctl_buffer_flush(fd);
1298         if (ret < 0) {
1299                 ERR("Fail to flush metadata buffers %d (ret: %d)", fd, ret);
1300         }
1301
1302         return 0;
1303 }
1304
1305 /*
1306  * Force flush buffer for channel.
1307  */
1308 int kernel_flush_buffer(struct ltt_kernel_channel *channel)
1309 {
1310         int ret;
1311         struct ltt_kernel_stream *stream;
1312
1313         assert(channel);
1314
1315         DBG("Flush buffer for channel %s", channel->channel->name);
1316
1317         cds_list_for_each_entry(stream, &channel->stream_list.head, list) {
1318                 DBG("Flushing channel stream %d", stream->fd);
1319                 ret = kernctl_buffer_flush(stream->fd);
1320                 if (ret < 0) {
1321                         PERROR("ioctl");
1322                         ERR("Fail to flush buffer for stream %d (ret: %d)",
1323                                         stream->fd, ret);
1324                 }
1325         }
1326
1327         return 0;
1328 }
1329
1330 /*
1331  * Stop tracing session.
1332  */
1333 int kernel_stop_session(struct ltt_kernel_session *session)
1334 {
1335         int ret;
1336
1337         assert(session);
1338
1339         ret = kernctl_stop_session(session->fd);
1340         if (ret < 0) {
1341                 goto error;
1342         }
1343
1344         DBG("Kernel session stopped");
1345
1346         return 0;
1347
1348 error:
1349         return ret;
1350 }
1351
1352 /*
1353  * Open stream of channel, register it to the kernel tracer and add it
1354  * to the stream list of the channel.
1355  *
1356  * Note: given that the streams may appear in random order wrt CPU
1357  * number (e.g. cpu hotplug), the index value of the stream number in
1358  * the stream name is not necessarily linked to the CPU number.
1359  *
1360  * Return the number of created stream. Else, a negative value.
1361  */
1362 int kernel_open_channel_stream(struct ltt_kernel_channel *channel)
1363 {
1364         int ret;
1365         struct ltt_kernel_stream *lks;
1366
1367         assert(channel);
1368
1369         while ((ret = kernctl_create_stream(channel->fd)) >= 0) {
1370                 lks = trace_kernel_create_stream(channel->channel->name,
1371                                 channel->stream_count);
1372                 if (lks == NULL) {
1373                         ret = close(ret);
1374                         if (ret) {
1375                                 PERROR("close");
1376                         }
1377                         goto error;
1378                 }
1379
1380                 lks->fd = ret;
1381                 /* Prevent fd duplication after execlp() */
1382                 ret = fcntl(lks->fd, F_SETFD, FD_CLOEXEC);
1383                 if (ret < 0) {
1384                         PERROR("fcntl session fd");
1385                 }
1386
1387                 lks->tracefile_size = channel->channel->attr.tracefile_size;
1388                 lks->tracefile_count = channel->channel->attr.tracefile_count;
1389
1390                 /* Add stream to channel stream list */
1391                 cds_list_add(&lks->list, &channel->stream_list.head);
1392                 channel->stream_count++;
1393
1394                 DBG("Kernel stream %s created (fd: %d, state: %d)", lks->name, lks->fd,
1395                                 lks->state);
1396         }
1397
1398         return channel->stream_count;
1399
1400 error:
1401         return -1;
1402 }
1403
1404 /*
1405  * Open the metadata stream and set it to the kernel session.
1406  */
1407 int kernel_open_metadata_stream(struct ltt_kernel_session *session)
1408 {
1409         int ret;
1410
1411         assert(session);
1412
1413         ret = kernctl_create_stream(session->metadata->fd);
1414         if (ret < 0) {
1415                 PERROR("kernel create metadata stream");
1416                 goto error;
1417         }
1418
1419         DBG("Kernel metadata stream created (fd: %d)", ret);
1420         session->metadata_stream_fd = ret;
1421         /* Prevent fd duplication after execlp() */
1422         ret = fcntl(session->metadata_stream_fd, F_SETFD, FD_CLOEXEC);
1423         if (ret < 0) {
1424                 PERROR("fcntl session fd");
1425         }
1426
1427         return 0;
1428
1429 error:
1430         return -1;
1431 }
1432
1433 /*
1434  * Get the event list from the kernel tracer and return the number of elements.
1435  */
1436 ssize_t kernel_list_events(struct lttng_event **events)
1437 {
1438         int fd, ret;
1439         char *event;
1440         size_t nbmem, count = 0;
1441         FILE *fp;
1442         struct lttng_event *elist;
1443
1444         assert(events);
1445
1446         fd = kernctl_tracepoint_list(kernel_tracer_fd);
1447         if (fd < 0) {
1448                 PERROR("kernel tracepoint list");
1449                 goto error;
1450         }
1451
1452         fp = fdopen(fd, "r");
1453         if (fp == NULL) {
1454                 PERROR("kernel tracepoint list fdopen");
1455                 goto error_fp;
1456         }
1457
1458         /*
1459          * Init memory size counter
1460          * See kernel-ctl.h for explanation of this value
1461          */
1462         nbmem = KERNEL_EVENT_INIT_LIST_SIZE;
1463         elist = zmalloc(sizeof(struct lttng_event) * nbmem);
1464         if (elist == NULL) {
1465                 PERROR("alloc list events");
1466                 count = -ENOMEM;
1467                 goto end;
1468         }
1469
1470         while (fscanf(fp, "event { name = %m[^;]; };\n", &event) == 1) {
1471                 if (count >= nbmem) {
1472                         struct lttng_event *new_elist;
1473                         size_t new_nbmem;
1474
1475                         new_nbmem = nbmem << 1;
1476                         DBG("Reallocating event list from %zu to %zu bytes",
1477                                         nbmem, new_nbmem);
1478                         new_elist = realloc(elist, new_nbmem * sizeof(struct lttng_event));
1479                         if (new_elist == NULL) {
1480                                 PERROR("realloc list events");
1481                                 free(event);
1482                                 free(elist);
1483                                 count = -ENOMEM;
1484                                 goto end;
1485                         }
1486                         /* Zero the new memory */
1487                         memset(new_elist + nbmem, 0,
1488                                 (new_nbmem - nbmem) * sizeof(struct lttng_event));
1489                         nbmem = new_nbmem;
1490                         elist = new_elist;
1491                 }
1492                 strncpy(elist[count].name, event, LTTNG_SYMBOL_NAME_LEN);
1493                 elist[count].name[LTTNG_SYMBOL_NAME_LEN - 1] = '\0';
1494                 elist[count].enabled = -1;
1495                 count++;
1496                 free(event);
1497         }
1498
1499         *events = elist;
1500         DBG("Kernel list events done (%zu events)", count);
1501 end:
1502         ret = fclose(fp);       /* closes both fp and fd */
1503         if (ret) {
1504                 PERROR("fclose");
1505         }
1506         return count;
1507
1508 error_fp:
1509         ret = close(fd);
1510         if (ret) {
1511                 PERROR("close");
1512         }
1513 error:
1514         return -1;
1515 }
1516
1517 /*
1518  * Get kernel version and validate it.
1519  */
1520 int kernel_validate_version(struct lttng_kernel_abi_tracer_version *version,
1521                 struct lttng_kernel_abi_tracer_abi_version *abi_version)
1522 {
1523         int ret;
1524
1525         ret = kernctl_tracer_version(kernel_tracer_fd, version);
1526         if (ret < 0) {
1527                 ERR("Failed to retrieve the lttng-modules version");
1528                 goto error;
1529         }
1530
1531         /* Validate version */
1532         if (version->major != VERSION_MAJOR) {
1533                 ERR("Kernel tracer major version (%d) is not compatible with lttng-tools major version (%d)",
1534                         version->major, VERSION_MAJOR);
1535                 goto error_version;
1536         }
1537         ret = kernctl_tracer_abi_version(kernel_tracer_fd, abi_version);
1538         if (ret < 0) {
1539                 ERR("Failed to retrieve lttng-modules ABI version");
1540                 goto error;
1541         }
1542         if (abi_version->major != LTTNG_KERNEL_ABI_MAJOR_VERSION) {
1543                 ERR("Kernel tracer ABI version (%d.%d) does not match the expected ABI major version (%d.*)",
1544                         abi_version->major, abi_version->minor,
1545                         LTTNG_KERNEL_ABI_MAJOR_VERSION);
1546                 goto error;
1547         }
1548         DBG2("Kernel tracer version validated (%d.%d, ABI %d.%d)",
1549                         version->major, version->minor,
1550                         abi_version->major, abi_version->minor);
1551         return 0;
1552
1553 error_version:
1554         ret = -1;
1555
1556 error:
1557         ERR("Kernel tracer version check failed; kernel tracing will not be available");
1558         return ret;
1559 }
1560
1561 /*
1562  * Kernel work-arounds called at the start of sessiond main().
1563  */
1564 int init_kernel_workarounds(void)
1565 {
1566         int ret;
1567         FILE *fp;
1568
1569         /*
1570          * boot_id needs to be read once before being used concurrently
1571          * to deal with a Linux kernel race. A fix is proposed for
1572          * upstream, but the work-around is needed for older kernels.
1573          */
1574         fp = fopen("/proc/sys/kernel/random/boot_id", "r");
1575         if (!fp) {
1576                 goto end_boot_id;
1577         }
1578         while (!feof(fp)) {
1579                 char buf[37] = "";
1580
1581                 ret = fread(buf, 1, sizeof(buf), fp);
1582                 if (ret < 0) {
1583                         /* Ignore error, we don't really care */
1584                 }
1585         }
1586         ret = fclose(fp);
1587         if (ret) {
1588                 PERROR("fclose");
1589         }
1590 end_boot_id:
1591         return 0;
1592 }
1593
1594 /*
1595  * Teardown of a kernel session, keeping data required by destroy notifiers.
1596  */
1597 void kernel_destroy_session(struct ltt_kernel_session *ksess)
1598 {
1599         struct lttng_trace_chunk *trace_chunk;
1600
1601         if (ksess == NULL) {
1602                 DBG3("No kernel session when tearing down session");
1603                 return;
1604         }
1605
1606         DBG("Tearing down kernel session");
1607         trace_chunk = ksess->current_trace_chunk;
1608
1609         /*
1610          * Destroy channels on the consumer if at least one FD has been sent and we
1611          * are in no output mode because the streams are in *no* monitor mode so we
1612          * have to send a command to clean them up or else they leaked.
1613          */
1614         if (!ksess->output_traces && ksess->consumer_fds_sent) {
1615                 int ret;
1616                 struct consumer_socket *socket;
1617                 struct lttng_ht_iter iter;
1618
1619                 /* For each consumer socket. */
1620                 rcu_read_lock();
1621                 cds_lfht_for_each_entry(ksess->consumer->socks->ht, &iter.iter,
1622                                 socket, node.node) {
1623                         struct ltt_kernel_channel *chan;
1624
1625                         /* For each channel, ask the consumer to destroy it. */
1626                         cds_list_for_each_entry(chan, &ksess->channel_list.head, list) {
1627                                 ret = kernel_consumer_destroy_channel(socket, chan);
1628                                 if (ret < 0) {
1629                                         /* Consumer is probably dead. Use next socket. */
1630                                         continue;
1631                                 }
1632                         }
1633                 }
1634                 rcu_read_unlock();
1635         }
1636
1637         /* Close any relayd session */
1638         consumer_output_send_destroy_relayd(ksess->consumer);
1639
1640         trace_kernel_destroy_session(ksess);
1641         lttng_trace_chunk_put(trace_chunk);
1642 }
1643
1644 /* Teardown of data required by destroy notifiers. */
1645 void kernel_free_session(struct ltt_kernel_session *ksess)
1646 {
1647         if (ksess == NULL) {
1648                 return;
1649         }
1650         trace_kernel_free_session(ksess);
1651 }
1652
1653 /*
1654  * Destroy a kernel channel object. It does not do anything on the tracer side.
1655  */
1656 void kernel_destroy_channel(struct ltt_kernel_channel *kchan)
1657 {
1658         struct ltt_kernel_session *ksess = NULL;
1659
1660         assert(kchan);
1661         assert(kchan->channel);
1662
1663         DBG3("Kernel destroy channel %s", kchan->channel->name);
1664
1665         /* Update channel count of associated session. */
1666         if (kchan->session) {
1667                 /* Keep pointer reference so we can update it after the destroy. */
1668                 ksess = kchan->session;
1669         }
1670
1671         trace_kernel_destroy_channel(kchan);
1672
1673         /*
1674          * At this point the kernel channel is not visible anymore. This is safe
1675          * since in order to work on a visible kernel session, the tracing session
1676          * lock (ltt_session.lock) MUST be acquired.
1677          */
1678         if (ksess) {
1679                 ksess->channel_count--;
1680         }
1681 }
1682
1683 /*
1684  * Take a snapshot for a given kernel session.
1685  *
1686  * Return LTTNG_OK on success or else return a LTTNG_ERR code.
1687  */
1688 enum lttng_error_code kernel_snapshot_record(
1689                 struct ltt_kernel_session *ksess,
1690                 const struct consumer_output *output, int wait,
1691                 uint64_t nb_packets_per_stream)
1692 {
1693         int err, ret, saved_metadata_fd;
1694         enum lttng_error_code status = LTTNG_OK;
1695         struct consumer_socket *socket;
1696         struct lttng_ht_iter iter;
1697         struct ltt_kernel_metadata *saved_metadata;
1698         char *trace_path = NULL;
1699         size_t consumer_path_offset = 0;
1700
1701         assert(ksess);
1702         assert(ksess->consumer);
1703         assert(output);
1704
1705         DBG("Kernel snapshot record started");
1706
1707         /* Save current metadata since the following calls will change it. */
1708         saved_metadata = ksess->metadata;
1709         saved_metadata_fd = ksess->metadata_stream_fd;
1710
1711         rcu_read_lock();
1712
1713         ret = kernel_open_metadata(ksess);
1714         if (ret < 0) {
1715                 status = LTTNG_ERR_KERN_META_FAIL;
1716                 goto error;
1717         }
1718
1719         ret = kernel_open_metadata_stream(ksess);
1720         if (ret < 0) {
1721                 status = LTTNG_ERR_KERN_META_FAIL;
1722                 goto error_open_stream;
1723         }
1724
1725         trace_path = setup_channel_trace_path(ksess->consumer,
1726                         "", &consumer_path_offset);
1727         if (!trace_path) {
1728                 status = LTTNG_ERR_INVALID;
1729                 goto error;
1730         }
1731         /* Send metadata to consumer and snapshot everything. */
1732         cds_lfht_for_each_entry(output->socks->ht, &iter.iter,
1733                         socket, node.node) {
1734                 struct ltt_kernel_channel *chan;
1735
1736                 pthread_mutex_lock(socket->lock);
1737                 /* This stream must not be monitored by the consumer. */
1738                 ret = kernel_consumer_add_metadata(socket, ksess, 0);
1739                 pthread_mutex_unlock(socket->lock);
1740                 if (ret < 0) {
1741                         status = LTTNG_ERR_KERN_META_FAIL;
1742                         goto error_consumer;
1743                 }
1744
1745                 /* For each channel, ask the consumer to snapshot it. */
1746                 cds_list_for_each_entry(chan, &ksess->channel_list.head, list) {
1747                         status = consumer_snapshot_channel(socket, chan->key, output, 0,
1748                                         ksess->uid, ksess->gid,
1749                                         &trace_path[consumer_path_offset], wait,
1750                                         nb_packets_per_stream);
1751                         if (status != LTTNG_OK) {
1752                                 (void) kernel_consumer_destroy_metadata(socket,
1753                                                 ksess->metadata);
1754                                 goto error_consumer;
1755                         }
1756                 }
1757
1758                 /* Snapshot metadata, */
1759                 status = consumer_snapshot_channel(socket, ksess->metadata->key, output,
1760                                 1, ksess->uid, ksess->gid, &trace_path[consumer_path_offset],
1761                                 wait, 0);
1762                 if (status != LTTNG_OK) {
1763                         goto error_consumer;
1764                 }
1765
1766                 /*
1767                  * The metadata snapshot is done, ask the consumer to destroy it since
1768                  * it's not monitored on the consumer side.
1769                  */
1770                 (void) kernel_consumer_destroy_metadata(socket, ksess->metadata);
1771         }
1772
1773 error_consumer:
1774         /* Close newly opened metadata stream. It's now on the consumer side. */
1775         err = close(ksess->metadata_stream_fd);
1776         if (err < 0) {
1777                 PERROR("close snapshot kernel");
1778         }
1779
1780 error_open_stream:
1781         trace_kernel_destroy_metadata(ksess->metadata);
1782 error:
1783         /* Restore metadata state.*/
1784         ksess->metadata = saved_metadata;
1785         ksess->metadata_stream_fd = saved_metadata_fd;
1786         rcu_read_unlock();
1787         free(trace_path);
1788         return status;
1789 }
1790
1791 /*
1792  * Get the syscall mask array from the kernel tracer.
1793  *
1794  * Return 0 on success else a negative value. In both case, syscall_mask should
1795  * be freed.
1796  */
1797 int kernel_syscall_mask(int chan_fd, char **syscall_mask, uint32_t *nr_bits)
1798 {
1799         assert(syscall_mask);
1800         assert(nr_bits);
1801
1802         return kernctl_syscall_mask(chan_fd, syscall_mask, nr_bits);
1803 }
1804
1805 static
1806 int kernel_tracer_abi_greater_or_equal(unsigned int major, unsigned int minor)
1807 {
1808         int ret;
1809         struct lttng_kernel_abi_tracer_abi_version abi;
1810
1811         ret = kernctl_tracer_abi_version(kernel_tracer_fd, &abi);
1812         if (ret < 0) {
1813                 ERR("Failed to retrieve lttng-modules ABI version");
1814                 goto error;
1815         }
1816
1817         ret = abi.major > major || (abi.major == major && abi.minor >= minor);
1818 error:
1819         return ret;
1820 }
1821
1822 /*
1823  * Check for the support of the RING_BUFFER_SNAPSHOT_SAMPLE_POSITIONS via abi
1824  * version number.
1825  *
1826  * Return 1 on success, 0 when feature is not supported, negative value in case
1827  * of errors.
1828  */
1829 int kernel_supports_ring_buffer_snapshot_sample_positions(void)
1830 {
1831         /*
1832          * RING_BUFFER_SNAPSHOT_SAMPLE_POSITIONS was introduced in 2.3
1833          */
1834         return kernel_tracer_abi_greater_or_equal(2, 3);
1835 }
1836
1837 /*
1838  * Check for the support of the packet sequence number via abi version number.
1839  *
1840  * Return 1 on success, 0 when feature is not supported, negative value in case
1841  * of errors.
1842  */
1843 int kernel_supports_ring_buffer_packet_sequence_number(void)
1844 {
1845         /*
1846          * Packet sequence number was introduced in LTTng 2.8,
1847          * lttng-modules ABI 2.1.
1848          */
1849         return kernel_tracer_abi_greater_or_equal(2, 1);
1850 }
1851
1852 /*
1853  * Check for the support of event notifiers via abi version number.
1854  *
1855  * Return 1 on success, 0 when feature is not supported, negative value in case
1856  * of errors.
1857  */
1858 int kernel_supports_event_notifiers(void)
1859 {
1860         /*
1861          * Event notifiers were introduced in LTTng 2.13, lttng-modules ABI 2.6.
1862          */
1863         return kernel_tracer_abi_greater_or_equal(2, 6);
1864 }
1865
1866 /*
1867  * Rotate a kernel session.
1868  *
1869  * Return LTTNG_OK on success or else an LTTng error code.
1870  */
1871 enum lttng_error_code kernel_rotate_session(struct ltt_session *session)
1872 {
1873         int ret;
1874         enum lttng_error_code status = LTTNG_OK;
1875         struct consumer_socket *socket;
1876         struct lttng_ht_iter iter;
1877         struct ltt_kernel_session *ksess = session->kernel_session;
1878
1879         assert(ksess);
1880         assert(ksess->consumer);
1881
1882         DBG("Rotate kernel session %s started (session %" PRIu64 ")",
1883                         session->name, session->id);
1884
1885         rcu_read_lock();
1886
1887         /*
1888          * Note that this loop will end after one iteration given that there is
1889          * only one kernel consumer.
1890          */
1891         cds_lfht_for_each_entry(ksess->consumer->socks->ht, &iter.iter,
1892                         socket, node.node) {
1893                 struct ltt_kernel_channel *chan;
1894
1895                 /* For each channel, ask the consumer to rotate it. */
1896                 cds_list_for_each_entry(chan, &ksess->channel_list.head, list) {
1897                         DBG("Rotate kernel channel %" PRIu64 ", session %s",
1898                                         chan->key, session->name);
1899                         ret = consumer_rotate_channel(socket, chan->key,
1900                                         ksess->uid, ksess->gid, ksess->consumer,
1901                                         /* is_metadata_channel */ false);
1902                         if (ret < 0) {
1903                                 status = LTTNG_ERR_ROTATION_FAIL_CONSUMER;
1904                                 goto error;
1905                         }
1906                 }
1907
1908                 /*
1909                  * Rotate the metadata channel.
1910                  */
1911                 ret = consumer_rotate_channel(socket, ksess->metadata->key,
1912                                 ksess->uid, ksess->gid, ksess->consumer,
1913                                 /* is_metadata_channel */ true);
1914                 if (ret < 0) {
1915                         status = LTTNG_ERR_ROTATION_FAIL_CONSUMER;
1916                         goto error;
1917                 }
1918         }
1919
1920 error:
1921         rcu_read_unlock();
1922         return status;
1923 }
1924
1925 enum lttng_error_code kernel_create_channel_subdirectories(
1926                 const struct ltt_kernel_session *ksess)
1927 {
1928         enum lttng_error_code ret = LTTNG_OK;
1929         enum lttng_trace_chunk_status chunk_status;
1930
1931         rcu_read_lock();
1932         assert(ksess->current_trace_chunk);
1933
1934         /*
1935          * Create the index subdirectory which will take care
1936          * of implicitly creating the channel's path.
1937          */
1938         chunk_status = lttng_trace_chunk_create_subdirectory(
1939                         ksess->current_trace_chunk,
1940                         DEFAULT_KERNEL_TRACE_DIR "/" DEFAULT_INDEX_DIR);
1941         if (chunk_status != LTTNG_TRACE_CHUNK_STATUS_OK) {
1942                 ret = LTTNG_ERR_CREATE_DIR_FAIL;
1943                 goto error;
1944         }
1945 error:
1946         rcu_read_unlock();
1947         return ret;
1948 }
1949
1950 /*
1951  * Setup necessary data for kernel tracer action.
1952  */
1953 LTTNG_HIDDEN
1954 int init_kernel_tracer(void)
1955 {
1956         int ret;
1957         bool is_root = !getuid();
1958
1959         /* Modprobe lttng kernel modules */
1960         ret = modprobe_lttng_control();
1961         if (ret < 0) {
1962                 goto error;
1963         }
1964
1965         /* Open debugfs lttng */
1966         kernel_tracer_fd = open(module_proc_lttng, O_RDWR);
1967         if (kernel_tracer_fd < 0) {
1968                 DBG("Failed to open %s", module_proc_lttng);
1969                 goto error_open;
1970         }
1971
1972         /* Validate kernel version */
1973         ret = kernel_validate_version(&the_kernel_tracer_version,
1974                         &the_kernel_tracer_abi_version);
1975         if (ret < 0) {
1976                 goto error_version;
1977         }
1978
1979         ret = modprobe_lttng_data();
1980         if (ret < 0) {
1981                 goto error_modules;
1982         }
1983
1984         ret = kernel_supports_ring_buffer_snapshot_sample_positions();
1985         if (ret < 0) {
1986                 goto error_modules;
1987         }
1988         if (ret < 1) {
1989                 WARN("Kernel tracer does not support buffer monitoring. "
1990                         "The monitoring timer of channels in the kernel domain "
1991                         "will be set to 0 (disabled).");
1992         }
1993
1994         ret = kernel_supports_event_notifiers();
1995         if (ret < 0) {
1996                 ERR("Failed to check for kernel tracer event notifier support");
1997                 goto error_modules;
1998         }
1999         ret = kernel_create_event_notifier_group(&kernel_tracer_event_notifier_group_fd);
2000         if (ret < 0) {
2001                 /* This is not fatal. */
2002                 WARN("Failed to create kernel event notifier group");
2003                 kernel_tracer_event_notifier_group_fd = -1;
2004         } else {
2005                 enum event_notifier_error_accounting_status error_accounting_status;
2006                 enum lttng_error_code error_code_ret =
2007                                 kernel_create_event_notifier_group_notification_fd(
2008                                                 &kernel_tracer_event_notifier_group_notification_fd);
2009
2010                 if (error_code_ret != LTTNG_OK) {
2011                         goto error_modules;
2012                 }
2013
2014                 error_accounting_status = event_notifier_error_accounting_register_kernel(
2015                                 kernel_tracer_event_notifier_group_fd);
2016                 if (error_accounting_status != EVENT_NOTIFIER_ERROR_ACCOUNTING_STATUS_OK) {
2017                         ERR("Failed to initialize event notifier error accounting for kernel tracer");
2018                         error_code_ret = LTTNG_ERR_EVENT_NOTIFIER_ERROR_ACCOUNTING;
2019                         goto error_modules;
2020                 }
2021
2022                 kernel_token_to_event_notifier_rule_ht = cds_lfht_new(
2023                                 DEFAULT_HT_SIZE, 1, 0,
2024                                 CDS_LFHT_AUTO_RESIZE | CDS_LFHT_ACCOUNTING,
2025                                 NULL);
2026                 if (!kernel_token_to_event_notifier_rule_ht) {
2027                         goto error_token_ht;
2028                 }
2029         }
2030
2031         DBG("Kernel tracer initialized: kernel tracer fd = %d, event notifier group fd = %d, event notifier group notification fd = %d",
2032                         kernel_tracer_fd, kernel_tracer_event_notifier_group_fd,
2033                         kernel_tracer_event_notifier_group_notification_fd);
2034
2035         ret = syscall_init_table(kernel_tracer_fd);
2036         if (ret < 0) {
2037                 ERR("Unable to populate syscall table. Syscall tracing won't "
2038                         "work for this session daemon.");
2039         }
2040
2041         return 0;
2042
2043 error_version:
2044         modprobe_remove_lttng_control();
2045         ret = close(kernel_tracer_fd);
2046         if (ret) {
2047                 PERROR("Failed to close kernel tracer file descriptor: fd = %d",
2048                                 kernel_tracer_fd);
2049         }
2050
2051         kernel_tracer_fd = -1;
2052         return LTTNG_ERR_KERN_VERSION;
2053
2054
2055 error_token_ht:
2056         ret = close(kernel_tracer_event_notifier_group_notification_fd);
2057         if (ret) {
2058                 PERROR("Failed to close kernel tracer event notifier group notification file descriptor: fd = %d",
2059                                 kernel_tracer_event_notifier_group_notification_fd);
2060         }
2061
2062         kernel_tracer_event_notifier_group_notification_fd = -1;
2063
2064 error_modules:
2065         ret = close(kernel_tracer_event_notifier_group_fd);
2066         if (ret) {
2067                 PERROR("Failed to close kernel tracer event notifier group file descriptor: fd = %d",
2068                                 kernel_tracer_event_notifier_group_fd);
2069         }
2070
2071         kernel_tracer_event_notifier_group_fd = -1;
2072
2073         ret = close(kernel_tracer_fd);
2074         if (ret) {
2075                 PERROR("Failed to close kernel tracer file descriptor: fd = %d",
2076                                 kernel_tracer_fd);
2077         }
2078
2079         kernel_tracer_fd = -1;
2080
2081 error_open:
2082         modprobe_remove_lttng_control();
2083
2084 error:
2085         WARN("No kernel tracer available");
2086         kernel_tracer_fd = -1;
2087         if (!is_root) {
2088                 return LTTNG_ERR_NEED_ROOT_SESSIOND;
2089         } else {
2090                 return LTTNG_ERR_KERN_NA;
2091         }
2092 }
2093
2094 LTTNG_HIDDEN
2095 void cleanup_kernel_tracer(void)
2096 {
2097         DBG2("Closing kernel event notifier group notification file descriptor");
2098         if (kernel_tracer_event_notifier_group_notification_fd >= 0) {
2099                 int ret = notification_thread_command_remove_tracer_event_source(
2100                                 the_notification_thread_handle,
2101                                 kernel_tracer_event_notifier_group_notification_fd);
2102                 if (ret != LTTNG_OK) {
2103                         ERR("Failed to remove kernel event notifier notification from notification thread");
2104                 }
2105
2106                 ret = close(kernel_tracer_event_notifier_group_notification_fd);
2107                 if (ret) {
2108                         PERROR("Failed to close kernel event notifier group notification file descriptor: fd = %d",
2109                                         kernel_tracer_event_notifier_group_notification_fd);
2110                 }
2111
2112                 kernel_tracer_event_notifier_group_notification_fd = -1;
2113         }
2114
2115         if (kernel_token_to_event_notifier_rule_ht) {
2116                 const int ret = cds_lfht_destroy(
2117                                 kernel_token_to_event_notifier_rule_ht, NULL);
2118                 assert(ret == 0);
2119         }
2120
2121         DBG2("Closing kernel event notifier group file descriptor");
2122         if (kernel_tracer_event_notifier_group_fd >= 0) {
2123                 const int ret = close(kernel_tracer_event_notifier_group_fd);
2124
2125                 if (ret) {
2126                         PERROR("Failed to close kernel event notifier group file descriptor: fd = %d",
2127                                         kernel_tracer_event_notifier_group_fd);
2128                 }
2129
2130                 kernel_tracer_event_notifier_group_fd = -1;
2131         }
2132
2133         DBG2("Closing kernel fd");
2134         if (kernel_tracer_fd >= 0) {
2135                 const int ret = close(kernel_tracer_fd);
2136
2137                 if (ret) {
2138                         PERROR("Failed to close kernel tracer file descriptor: fd = %d",
2139                                         kernel_tracer_fd);
2140                 }
2141
2142                 kernel_tracer_fd = -1;
2143         }
2144
2145         free(syscall_table);
2146 }
2147
2148 LTTNG_HIDDEN
2149 bool kernel_tracer_is_initialized(void)
2150 {
2151         return kernel_tracer_fd >= 0;
2152 }
2153
2154 /*
2155  *  Clear a kernel session.
2156  *
2157  * Return LTTNG_OK on success or else an LTTng error code.
2158  */
2159 enum lttng_error_code kernel_clear_session(struct ltt_session *session)
2160 {
2161         int ret;
2162         enum lttng_error_code status = LTTNG_OK;
2163         struct consumer_socket *socket;
2164         struct lttng_ht_iter iter;
2165         struct ltt_kernel_session *ksess = session->kernel_session;
2166
2167         assert(ksess);
2168         assert(ksess->consumer);
2169
2170         DBG("Clear kernel session %s (session %" PRIu64 ")",
2171                         session->name, session->id);
2172
2173         rcu_read_lock();
2174
2175         if (ksess->active) {
2176                 ERR("Expecting inactive session %s (%" PRIu64 ")", session->name, session->id);
2177                 status = LTTNG_ERR_FATAL;
2178                 goto end;
2179         }
2180
2181         /*
2182          * Note that this loop will end after one iteration given that there is
2183          * only one kernel consumer.
2184          */
2185         cds_lfht_for_each_entry(ksess->consumer->socks->ht, &iter.iter,
2186                         socket, node.node) {
2187                 struct ltt_kernel_channel *chan;
2188
2189                 /* For each channel, ask the consumer to clear it. */
2190                 cds_list_for_each_entry(chan, &ksess->channel_list.head, list) {
2191                         DBG("Clear kernel channel %" PRIu64 ", session %s",
2192                                         chan->key, session->name);
2193                         ret = consumer_clear_channel(socket, chan->key);
2194                         if (ret < 0) {
2195                                 goto error;
2196                         }
2197                 }
2198
2199                 if (!ksess->metadata) {
2200                         /*
2201                          * Nothing to do for the metadata.
2202                          * This is a snapshot session.
2203                          * The metadata is genererated on the fly.
2204                          */
2205                         continue;
2206                 }
2207
2208                 /*
2209                  * Clear the metadata channel.
2210                  * Metadata channel is not cleared per se but we still need to
2211                  * perform a rotation operation on it behind the scene.
2212                  */
2213                 ret = consumer_clear_channel(socket, ksess->metadata->key);
2214                 if (ret < 0) {
2215                         goto error;
2216                 }
2217         }
2218
2219         goto end;
2220 error:
2221         switch (-ret) {
2222         case LTTCOMM_CONSUMERD_RELAYD_CLEAR_DISALLOWED:
2223               status = LTTNG_ERR_CLEAR_RELAY_DISALLOWED;
2224               break;
2225         default:
2226               status = LTTNG_ERR_CLEAR_FAIL_CONSUMER;
2227               break;
2228         }
2229 end:
2230         rcu_read_unlock();
2231         return status;
2232 }
2233
2234 enum lttng_error_code kernel_create_event_notifier_group_notification_fd(
2235                 int *event_notifier_group_notification_fd)
2236 {
2237         int local_fd = -1, ret;
2238         enum lttng_error_code error_code_ret;
2239
2240         assert(event_notifier_group_notification_fd);
2241
2242         ret = kernctl_create_event_notifier_group_notification_fd(
2243                         kernel_tracer_event_notifier_group_fd);
2244         if (ret < 0) {
2245                 PERROR("Failed to create kernel event notifier group notification file descriptor");
2246                 error_code_ret = LTTNG_ERR_EVENT_NOTIFIER_GROUP_NOTIFICATION_FD;
2247                 goto error;
2248         }
2249
2250         local_fd = ret;
2251
2252         /* Prevent fd duplication after execlp(). */
2253         ret = fcntl(local_fd, F_SETFD, FD_CLOEXEC);
2254         if (ret < 0) {
2255                 PERROR("Failed to set FD_CLOEXEC on kernel event notifier group notification file descriptor: fd = %d",
2256                                 local_fd);
2257                 error_code_ret = LTTNG_ERR_EVENT_NOTIFIER_GROUP_NOTIFICATION_FD;
2258                 goto error;
2259         }
2260
2261         DBG("Created kernel notifier group notification file descriptor: fd = %d",
2262                         local_fd);
2263         error_code_ret = LTTNG_OK;
2264         *event_notifier_group_notification_fd = local_fd;
2265         local_fd = -1;
2266
2267 error:
2268         if (local_fd >= 0) {
2269                 ret = close(local_fd);
2270                 if (ret) {
2271                         PERROR("Failed to close kernel event notifier group notification file descriptor: fd = %d",
2272                                         local_fd);
2273                 }
2274         }
2275
2276         return error_code_ret;
2277 }
2278
2279 enum lttng_error_code kernel_destroy_event_notifier_group_notification_fd(
2280                 int event_notifier_group_notification_fd)
2281 {
2282         enum lttng_error_code ret_code = LTTNG_OK;
2283
2284         DBG("Closing event notifier group notification file descriptor: fd = %d",
2285                         event_notifier_group_notification_fd);
2286         if (event_notifier_group_notification_fd >= 0) {
2287                 const int ret = close(event_notifier_group_notification_fd);
2288                 if (ret) {
2289                         PERROR("Failed to close event notifier group notification file descriptor: fd = %d",
2290                                         event_notifier_group_notification_fd);
2291                 }
2292         }
2293
2294         return ret_code;
2295 }
2296
2297 static
2298 unsigned long hash_trigger(const struct lttng_trigger *trigger)
2299 {
2300         const struct lttng_condition *condition =
2301                         lttng_trigger_get_const_condition(trigger);
2302
2303         return lttng_condition_hash(condition);
2304 }
2305
2306 static
2307 int match_trigger(struct cds_lfht_node *node, const void *key)
2308 {
2309         const struct ltt_kernel_event_notifier_rule *event_notifier_rule;
2310         const struct lttng_trigger *trigger = key;
2311
2312         event_notifier_rule = caa_container_of(node,
2313                         const struct ltt_kernel_event_notifier_rule, ht_node);
2314
2315         return lttng_trigger_is_equal(trigger, event_notifier_rule->trigger);
2316 }
2317
2318 static enum lttng_error_code kernel_create_event_notifier_rule(
2319                 struct lttng_trigger *trigger,
2320                 const struct lttng_credentials *creds, uint64_t token)
2321 {
2322         int err, fd, ret = 0;
2323         enum lttng_error_code error_code_ret;
2324         enum lttng_condition_status condition_status;
2325         enum lttng_condition_type condition_type;
2326         enum lttng_event_rule_type event_rule_type;
2327         struct ltt_kernel_event_notifier_rule *event_notifier_rule;
2328         struct lttng_kernel_abi_event_notifier kernel_event_notifier = {};
2329         unsigned int capture_bytecode_count = 0, i;
2330         const struct lttng_condition *condition = NULL;
2331         const struct lttng_event_rule *event_rule = NULL;
2332         enum lttng_condition_status cond_status;
2333
2334         assert(trigger);
2335
2336         condition = lttng_trigger_get_const_condition(trigger);
2337         assert(condition);
2338
2339         condition_type = lttng_condition_get_type(condition);
2340         assert(condition_type == LTTNG_CONDITION_TYPE_EVENT_RULE_MATCHES);
2341
2342         /* Does not acquire a reference. */
2343         condition_status = lttng_condition_event_rule_matches_get_rule(
2344                         condition, &event_rule);
2345         assert(condition_status == LTTNG_CONDITION_STATUS_OK);
2346         assert(event_rule);
2347
2348         event_rule_type = lttng_event_rule_get_type(event_rule);
2349         assert(event_rule_type != LTTNG_EVENT_RULE_TYPE_UNKNOWN);
2350
2351         error_code_ret = trace_kernel_create_event_notifier_rule(trigger, token,
2352                         lttng_condition_event_rule_matches_get_error_counter_index(
2353                                         condition),
2354                         &event_notifier_rule);
2355         if (error_code_ret != LTTNG_OK) {
2356                 goto error;
2357         }
2358
2359         error_code_ret = trace_kernel_init_event_notifier_from_event_rule(
2360                         event_rule, &kernel_event_notifier);
2361         if (error_code_ret != LTTNG_OK) {
2362                 goto free_event;
2363         }
2364
2365         kernel_event_notifier.event.token = event_notifier_rule->token;
2366         kernel_event_notifier.error_counter_idx =
2367                         lttng_condition_event_rule_matches_get_error_counter_index(
2368                                         condition);
2369
2370         fd = kernctl_create_event_notifier(
2371                         kernel_tracer_event_notifier_group_fd,
2372                         &kernel_event_notifier);
2373         if (fd < 0) {
2374                 switch (-fd) {
2375                 case EEXIST:
2376                         error_code_ret = LTTNG_ERR_KERN_EVENT_EXIST;
2377                         break;
2378                 case ENOSYS:
2379                         WARN("Failed to create kernel event notifier: not notifier type not implemented");
2380                         error_code_ret = LTTNG_ERR_KERN_EVENT_ENOSYS;
2381                         break;
2382                 case ENOENT:
2383                         WARN("Failed to create kernel event notifier: not found: name = '%s'",
2384                                         kernel_event_notifier.event.name);
2385                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2386                         break;
2387                 default:
2388                         PERROR("Failed to create kernel event notifier: error code = %d, name = '%s'",
2389                                         fd, kernel_event_notifier.event.name);
2390                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2391                 }
2392                 goto free_event;
2393         }
2394
2395         event_notifier_rule->fd = fd;
2396         /* Prevent fd duplication after execlp(). */
2397         err = fcntl(event_notifier_rule->fd, F_SETFD, FD_CLOEXEC);
2398         if (err < 0) {
2399                 PERROR("Failed to set FD_CLOEXEC on kernel event notifier file descriptor: fd = %d",
2400                                 fd);
2401                 error_code_ret = LTTNG_ERR_FATAL;
2402                 goto set_cloexec_error;
2403         }
2404
2405         if (event_notifier_rule->filter) {
2406                 err = kernctl_filter(event_notifier_rule->fd, event_notifier_rule->filter);
2407                 if (err < 0) {
2408                         switch (-err) {
2409                         case ENOMEM:
2410                                 error_code_ret = LTTNG_ERR_FILTER_NOMEM;
2411                                 break;
2412                         default:
2413                                 error_code_ret = LTTNG_ERR_FILTER_INVAL;
2414                                 break;
2415                         }
2416                         goto filter_error;
2417                 }
2418         }
2419
2420         if (lttng_event_rule_get_type(event_rule) ==
2421                         LTTNG_EVENT_RULE_TYPE_KERNEL_UPROBE) {
2422                 ret = userspace_probe_event_rule_add_callsites(
2423                                 event_rule, creds, event_notifier_rule->fd);
2424                 if (ret) {
2425                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2426                         goto add_callsite_error;
2427                 }
2428         }
2429
2430         /* Set the capture bytecode if any. */
2431         cond_status = lttng_condition_event_rule_matches_get_capture_descriptor_count(
2432                         condition, &capture_bytecode_count);
2433         assert(cond_status == LTTNG_CONDITION_STATUS_OK);
2434
2435         for (i = 0; i < capture_bytecode_count; i++) {
2436                 const struct lttng_bytecode *capture_bytecode =
2437                                 lttng_condition_event_rule_matches_get_capture_bytecode_at_index(
2438                                                 condition, i);
2439
2440                 if (capture_bytecode == NULL) {
2441                         ERR("Unexpected NULL capture bytecode on condition");
2442                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2443                         goto capture_error;
2444                 }
2445
2446                 ret = kernctl_capture(event_notifier_rule->fd, capture_bytecode);
2447                 if (ret < 0) {
2448                         ERR("Failed to set capture bytecode on event notifier rule fd: fd = %d",
2449                                         event_notifier_rule->fd);
2450                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2451                         goto capture_error;
2452                 }
2453         }
2454
2455         err = kernctl_enable(event_notifier_rule->fd);
2456         if (err < 0) {
2457                 switch (-err) {
2458                 case EEXIST:
2459                         error_code_ret = LTTNG_ERR_KERN_EVENT_EXIST;
2460                         break;
2461                 default:
2462                         PERROR("enable kernel event notifier");
2463                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2464                         break;
2465                 }
2466                 goto enable_error;
2467         }
2468
2469         /* Add trigger to kernel token mapping in the hash table. */
2470         rcu_read_lock();
2471         cds_lfht_add(kernel_token_to_event_notifier_rule_ht, hash_trigger(trigger),
2472                         &event_notifier_rule->ht_node);
2473         rcu_read_unlock();
2474
2475         DBG("Created kernel event notifier: name = '%s', fd = %d",
2476                         kernel_event_notifier.event.name,
2477                         event_notifier_rule->fd);
2478
2479         return LTTNG_OK;
2480
2481 capture_error:
2482 add_callsite_error:
2483 enable_error:
2484 set_cloexec_error:
2485 filter_error:
2486         {
2487                 const int close_ret = close(event_notifier_rule->fd);
2488
2489                 if (close_ret) {
2490                         PERROR("Failed to close kernel event notifier file descriptor: fd = %d",
2491                                         event_notifier_rule->fd);
2492                 }
2493         }
2494 free_event:
2495         free(event_notifier_rule);
2496 error:
2497         return error_code_ret;
2498 }
2499
2500 enum lttng_error_code kernel_register_event_notifier(
2501                 struct lttng_trigger *trigger,
2502                 const struct lttng_credentials *cmd_creds)
2503 {
2504         enum lttng_error_code ret;
2505         enum lttng_condition_status status;
2506         enum lttng_domain_type domain_type;
2507         const struct lttng_event_rule *event_rule;
2508         const struct lttng_condition *const condition =
2509                         lttng_trigger_get_const_condition(trigger);
2510         const uint64_t token = lttng_trigger_get_tracer_token(trigger);
2511
2512         assert(condition);
2513
2514         /* Does not acquire a reference to the event rule. */
2515         status = lttng_condition_event_rule_matches_get_rule(
2516                         condition, &event_rule);
2517         assert(status == LTTNG_CONDITION_STATUS_OK);
2518
2519         domain_type = lttng_event_rule_get_domain_type(event_rule);
2520         assert(domain_type == LTTNG_DOMAIN_KERNEL);
2521
2522         ret = kernel_create_event_notifier_rule(trigger, cmd_creds, token);
2523         if (ret != LTTNG_OK) {
2524                 ERR("Failed to create kernel event notifier rule");
2525         }
2526
2527         return ret;
2528 }
2529
2530 enum lttng_error_code kernel_unregister_event_notifier(
2531                 const struct lttng_trigger *trigger)
2532 {
2533         struct ltt_kernel_event_notifier_rule *token_event_rule_element;
2534         struct cds_lfht_node *node;
2535         struct cds_lfht_iter iter;
2536         enum lttng_error_code error_code_ret;
2537         int ret;
2538
2539         rcu_read_lock();
2540
2541         cds_lfht_lookup(kernel_token_to_event_notifier_rule_ht,
2542                         hash_trigger(trigger), match_trigger, trigger, &iter);
2543
2544         node = cds_lfht_iter_get_node(&iter);
2545         if (!node) {
2546                 error_code_ret = LTTNG_ERR_TRIGGER_NOT_FOUND;
2547                 goto error;
2548         }
2549
2550         token_event_rule_element = caa_container_of(node,
2551                         struct ltt_kernel_event_notifier_rule, ht_node);
2552
2553         ret = kernel_disable_event_notifier_rule(token_event_rule_element);
2554         if (ret) {
2555                 error_code_ret = LTTNG_ERR_FATAL;
2556                 goto error;
2557         }
2558
2559         trace_kernel_destroy_event_notifier_rule(token_event_rule_element);
2560         error_code_ret = LTTNG_OK;
2561
2562 error:
2563         rcu_read_unlock();
2564
2565         return error_code_ret;
2566 }
2567
2568 int kernel_get_notification_fd(void)
2569 {
2570         return kernel_tracer_event_notifier_group_notification_fd;
2571 }