src/bin/lttng-sessiond/kernel.cpp

   1 /*
   2  * Copyright (C) 2011 EfficiOS Inc.
   3  *
   4  * SPDX-License-Identifier: GPL-2.0-only
   5  *
   6  */
   7
   8 #define _LGPL_SOURCE
   9 #include <fcntl.h>
  10 #include <stdlib.h>
  11 #include <stdio.h>
  12 #include <string.h>
  13 #include <unistd.h>
  14 #include <inttypes.h>
  15 #include <sys/types.h>
  16
  17 #include <common/common.hpp>
  18 #include <common/hashtable/utils.hpp>
  19 #include <common/trace-chunk.hpp>
  20 #include <common/kernel-ctl/kernel-ctl.hpp>
  21 #include <common/kernel-ctl/kernel-ioctl.hpp>
  22 #include <common/sessiond-comm/sessiond-comm.hpp>
  23 #include <common/tracker.hpp>
  24 #include <common/utils.hpp>
  25 #include <lttng/event.h>
  26 #include <lttng/lttng-error.h>
  27 #include <lttng/tracker.h>
  28
  29 #include <lttng/userspace-probe.h>
  30 #include <lttng/userspace-probe-internal.hpp>
  31 #include <lttng/condition/event-rule-matches.h>
  32 #include <lttng/condition/event-rule-matches-internal.hpp>
  33 #include <lttng/event-rule/event-rule.h>
  34 #include <lttng/event-rule/event-rule-internal.hpp>
  35 #include <lttng/event-rule/kernel-uprobe-internal.hpp>
  36
  37 #include "event-notifier-error-accounting.hpp"
  38 #include "lttng-sessiond.hpp"
  39 #include "lttng-syscall.hpp"
  40 #include "condition-internal.hpp"
  41 #include "consumer.hpp"
  42 #include "kernel.hpp"
  43 #include "kernel-consumer.hpp"
  44 #include "kern-modules.hpp"
  45 #include "sessiond-config.hpp"
  46 #include "utils.hpp"
  47 #include "rotate.hpp"
  48 #include "modprobe.hpp"
  49 #include "tracker.hpp"
  50 #include "notification-thread-commands.hpp"
  51
  52 /*
  53  * Key used to reference a channel between the sessiond and the consumer. This
  54  * is only read and updated with the session_list lock held.
  55  */
  56 static uint64_t next_kernel_channel_key;
  57
  58 static const char *module_proc_lttng = "/proc/lttng";
  59
  60 static int kernel_tracer_fd = -1;
  61 static int kernel_tracer_event_notifier_group_fd = -1;
  62 static int kernel_tracer_event_notifier_group_notification_fd = -1;
  63 static struct cds_lfht *kernel_token_to_event_notifier_rule_ht;
  64
  65 /*
  66  * Add context on a kernel channel.
  67  *
  68  * Assumes the ownership of ctx.
  69  */
  70 int kernel_add_channel_context(struct ltt_kernel_channel *chan,
  71                 struct ltt_kernel_context *ctx)
  72 {
  73         int ret;
  74
  75         LTTNG_ASSERT(chan);
  76         LTTNG_ASSERT(ctx);
  77
  78         DBG("Adding context to channel %s", chan->channel->name);
  79         ret = kernctl_add_context(chan->fd, &ctx->ctx);
  80         if (ret < 0) {
  81                 switch (-ret) {
  82                 case ENOSYS:
  83                         /* Exists but not available for this kernel */
  84                         ret = LTTNG_ERR_KERN_CONTEXT_UNAVAILABLE;
  85                         goto error;
  86                 case EEXIST:
  87                         /* If EEXIST, we just ignore the error */
  88                         ret = 0;
  89                         goto end;
  90                 default:
  91                         PERROR("add context ioctl");
  92                         ret = LTTNG_ERR_KERN_CONTEXT_FAIL;
  93                         goto error;
  94                 }
  95         }
  96         ret = 0;
  97
  98 end:
  99         cds_list_add_tail(&ctx->list, &chan->ctx_list);
 100         ctx->in_list = true;
 101         ctx = NULL;
 102 error:
 103         if (ctx) {
 104                 trace_kernel_destroy_context(ctx);
 105         }
 106         return ret;
 107 }
 108
 109 /*
 110  * Create a new kernel session, register it to the kernel tracer and add it to
 111  * the session daemon session.
 112  */
 113 int kernel_create_session(struct ltt_session *session)
 114 {
 115         int ret;
 116         struct ltt_kernel_session *lks;
 117
 118         LTTNG_ASSERT(session);
 119
 120         /* Allocate data structure */
 121         lks = trace_kernel_create_session();
 122         if (lks == NULL) {
 123                 ret = -1;
 124                 goto error;
 125         }
 126
 127         /* Kernel tracer session creation */
 128         ret = kernctl_create_session(kernel_tracer_fd);
 129         if (ret < 0) {
 130                 PERROR("ioctl kernel create session");
 131                 goto error;
 132         }
 133
 134         lks->fd = ret;
 135         /* Prevent fd duplication after execlp() */
 136         ret = fcntl(lks->fd, F_SETFD, FD_CLOEXEC);
 137         if (ret < 0) {
 138                 PERROR("fcntl session fd");
 139         }
 140
 141         lks->id = session->id;
 142         lks->consumer_fds_sent = 0;
 143         session->kernel_session = lks;
 144
 145         DBG("Kernel session created (fd: %d)", lks->fd);
 146
 147         /*
 148          * This is necessary since the creation time is present in the session
 149          * name when it is generated.
 150          */
 151         if (session->has_auto_generated_name) {
 152                 ret = kernctl_session_set_name(lks->fd, DEFAULT_SESSION_NAME);
 153         } else {
 154                 ret = kernctl_session_set_name(lks->fd, session->name);
 155         }
 156         if (ret) {
 157                 WARN("Could not set kernel session name for session %" PRIu64 " name: %s",
 158                         session->id, session->name);
 159         }
 160
 161         ret = kernctl_session_set_creation_time(lks->fd, session->creation_time);
 162         if (ret) {
 163                 WARN("Could not set kernel session creation time for session %" PRIu64 " name: %s",
 164                         session->id, session->name);
 165         }
 166
 167         return 0;
 168
 169 error:
 170         if (lks) {
 171                 trace_kernel_destroy_session(lks);
 172                 trace_kernel_free_session(lks);
 173         }
 174         return ret;
 175 }
 176
 177 /*
 178  * Create a kernel channel, register it to the kernel tracer and add it to the
 179  * kernel session.
 180  */
 181 int kernel_create_channel(struct ltt_kernel_session *session,
 182                 struct lttng_channel *chan)
 183 {
 184         int ret;
 185         struct ltt_kernel_channel *lkc;
 186
 187         LTTNG_ASSERT(session);
 188         LTTNG_ASSERT(chan);
 189
 190         /* Allocate kernel channel */
 191         lkc = trace_kernel_create_channel(chan);
 192         if (lkc == NULL) {
 193                 goto error;
 194         }
 195
 196         DBG3("Kernel create channel %s with attr: %d, %" PRIu64 ", %" PRIu64 ", %u, %u, %d, %d",
 197                         chan->name, lkc->channel->attr.overwrite,
 198                         lkc->channel->attr.subbuf_size, lkc->channel->attr.num_subbuf,
 199                         lkc->channel->attr.switch_timer_interval, lkc->channel->attr.read_timer_interval,
 200                         lkc->channel->attr.live_timer_interval, lkc->channel->attr.output);
 201
 202         /* Kernel tracer channel creation */
 203         ret = kernctl_create_channel(session->fd, &lkc->channel->attr);
 204         if (ret < 0) {
 205                 PERROR("ioctl kernel create channel");
 206                 goto error;
 207         }
 208
 209         /* Setup the channel fd */
 210         lkc->fd = ret;
 211         /* Prevent fd duplication after execlp() */
 212         ret = fcntl(lkc->fd, F_SETFD, FD_CLOEXEC);
 213         if (ret < 0) {
 214                 PERROR("fcntl session fd");
 215         }
 216
 217         /* Add channel to session */
 218         cds_list_add(&lkc->list, &session->channel_list.head);
 219         session->channel_count++;
 220         lkc->session = session;
 221         lkc->key = ++next_kernel_channel_key;
 222
 223         DBG("Kernel channel %s created (fd: %d, key: %" PRIu64 ")",
 224                         lkc->channel->name, lkc->fd, lkc->key);
 225
 226         return 0;
 227
 228 error:
 229         if (lkc) {
 230                 free(lkc->channel);
 231                 free(lkc);
 232         }
 233         return -1;
 234 }
 235
 236 /*
 237  * Create a kernel event notifier group, register it to the kernel tracer and
 238  * add it to the kernel session.
 239  */
 240 static int kernel_create_event_notifier_group(int *event_notifier_group_fd)
 241 {
 242         int ret;
 243         int local_fd = -1;
 244
 245         LTTNG_ASSERT(event_notifier_group_fd);
 246
 247         /* Kernel event notifier group creation. */
 248         ret = kernctl_create_event_notifier_group(kernel_tracer_fd);
 249         if (ret < 0) {
 250                 PERROR("Failed to create kernel event notifier group");
 251                 ret = -1;
 252                 goto error;
 253         }
 254
 255         local_fd = ret;
 256
 257         /* Prevent fd duplication after execlp(). */
 258         ret = fcntl(local_fd, F_SETFD, FD_CLOEXEC);
 259         if (ret < 0) {
 260                 PERROR("Failed to set FD_CLOEXEC on kernel event notifier group file descriptor: fd = %d",
 261                                 local_fd);
 262                 goto error;
 263         }
 264
 265         DBG("Created kernel event notifier group: fd = %d", local_fd);
 266         *event_notifier_group_fd = local_fd;
 267         local_fd = -1;
 268         ret = 0;
 269 error:
 270         if (local_fd >= 0) {
 271                 ret = close(local_fd);
 272                 if (ret) {
 273                         PERROR("Failed to close kernel event notifier group file descriptor: fd = %d",
 274                                         local_fd);
 275                 }
 276         }
 277
 278         return ret;
 279 }
 280
 281 /*
 282  * Compute the offset of the instrumentation byte in the binary based on the
 283  * function probe location using the ELF lookup method.
 284  *
 285  * Returns 0 on success and set the offset out parameter to the offset of the
 286  * elf symbol
 287  * Returns -1 on error
 288  */
 289 static
 290 int extract_userspace_probe_offset_function_elf(
 291                 const struct lttng_userspace_probe_location *probe_location,
 292                 uid_t uid, gid_t gid, uint64_t *offset)
 293 {
 294         int fd;
 295         int ret = 0;
 296         const char *symbol = NULL;
 297         const struct lttng_userspace_probe_location_lookup_method *lookup = NULL;
 298         enum lttng_userspace_probe_location_lookup_method_type lookup_method_type;
 299
 300         LTTNG_ASSERT(lttng_userspace_probe_location_get_type(probe_location) ==
 301                         LTTNG_USERSPACE_PROBE_LOCATION_TYPE_FUNCTION);
 302
 303         lookup = lttng_userspace_probe_location_get_lookup_method(
 304                         probe_location);
 305         if (!lookup) {
 306                 ret = -1;
 307                 goto end;
 308         }
 309
 310         lookup_method_type =
 311                         lttng_userspace_probe_location_lookup_method_get_type(lookup);
 312
 313         LTTNG_ASSERT(lookup_method_type ==
 314                         LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_FUNCTION_ELF);
 315
 316         symbol = lttng_userspace_probe_location_function_get_function_name(
 317                         probe_location);
 318         if (!symbol) {
 319                 ret = -1;
 320                 goto end;
 321         }
 322
 323         fd = lttng_userspace_probe_location_function_get_binary_fd(probe_location);
 324         if (fd < 0) {
 325                 ret = -1;
 326                 goto end;
 327         }
 328
 329         ret = run_as_extract_elf_symbol_offset(fd, symbol, uid, gid, offset);
 330         if (ret < 0) {
 331                 DBG("userspace probe offset calculation failed for "
 332                                 "function %s", symbol);
 333                 goto end;
 334         }
 335
 336         DBG("userspace probe elf offset for %s is 0x%jd", symbol, (intmax_t)(*offset));
 337 end:
 338         return ret;
 339 }
 340
 341 /*
 342  * Compute the offsets of the instrumentation bytes in the binary based on the
 343  * tracepoint probe location using the SDT lookup method. This function
 344  * allocates the offsets buffer, the caller must free it.
 345  *
 346  * Returns 0 on success and set the offset out parameter to the offsets of the
 347  * SDT tracepoint.
 348  * Returns -1 on error.
 349  */
 350 static
 351 int extract_userspace_probe_offset_tracepoint_sdt(
 352                 const struct lttng_userspace_probe_location *probe_location,
 353                 uid_t uid, gid_t gid, uint64_t **offsets,
 354                 uint32_t *offsets_count)
 355 {
 356         enum lttng_userspace_probe_location_lookup_method_type lookup_method_type;
 357         const struct lttng_userspace_probe_location_lookup_method *lookup = NULL;
 358         const char *probe_name = NULL, *provider_name = NULL;
 359         int ret = 0;
 360         int fd, i;
 361
 362         LTTNG_ASSERT(lttng_userspace_probe_location_get_type(probe_location) ==
 363                         LTTNG_USERSPACE_PROBE_LOCATION_TYPE_TRACEPOINT);
 364
 365         lookup = lttng_userspace_probe_location_get_lookup_method(probe_location);
 366         if (!lookup) {
 367                 ret = -1;
 368                 goto end;
 369         }
 370
 371         lookup_method_type =
 372                         lttng_userspace_probe_location_lookup_method_get_type(lookup);
 373
 374         LTTNG_ASSERT(lookup_method_type ==
 375                         LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_TRACEPOINT_SDT);
 376
 377
 378         probe_name = lttng_userspace_probe_location_tracepoint_get_probe_name(
 379                         probe_location);
 380         if (!probe_name) {
 381                 ret = -1;
 382                 goto end;
 383         }
 384
 385         provider_name = lttng_userspace_probe_location_tracepoint_get_provider_name(
 386                         probe_location);
 387         if (!provider_name) {
 388                 ret = -1;
 389                 goto end;
 390         }
 391
 392         fd = lttng_userspace_probe_location_tracepoint_get_binary_fd(probe_location);
 393         if (fd < 0) {
 394                 ret = -1;
 395                 goto end;
 396         }
 397
 398         ret = run_as_extract_sdt_probe_offsets(fd, provider_name, probe_name,
 399                         uid, gid, offsets, offsets_count);
 400         if (ret < 0) {
 401                 DBG("userspace probe offset calculation failed for sdt "
 402                                 "probe %s:%s", provider_name, probe_name);
 403                 goto end;
 404         }
 405
 406         if (*offsets_count == 0) {
 407                 DBG("no userspace probe offset found");
 408                 goto end;
 409         }
 410
 411         DBG("%u userspace probe SDT offsets found for %s:%s at:",
 412                         *offsets_count, provider_name, probe_name);
 413         for (i = 0; i < *offsets_count; i++) {
 414                 DBG("\t0x%jd", (intmax_t)((*offsets)[i]));
 415         }
 416 end:
 417         return ret;
 418 }
 419
 420 static
 421 int userspace_probe_add_callsite(
 422                 const struct lttng_userspace_probe_location *location,
 423                 uid_t uid, gid_t gid, int fd)
 424 {
 425         const struct lttng_userspace_probe_location_lookup_method *lookup_method = NULL;
 426         enum lttng_userspace_probe_location_lookup_method_type type;
 427         int ret;
 428
 429         lookup_method = lttng_userspace_probe_location_get_lookup_method(location);
 430         if (!lookup_method) {
 431                 ret = -1;
 432                 goto end;
 433         }
 434
 435         type = lttng_userspace_probe_location_lookup_method_get_type(lookup_method);
 436         switch (type) {
 437         case LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_FUNCTION_ELF:
 438         {
 439                 struct lttng_kernel_abi_event_callsite callsite;
 440                 uint64_t offset;
 441
 442                 ret = extract_userspace_probe_offset_function_elf(location,
 443                                 uid, gid, &offset);
 444                 if (ret) {
 445                         ret = LTTNG_ERR_PROBE_LOCATION_INVAL;
 446                         goto end;
 447                 }
 448
 449                 callsite.u.uprobe.offset = offset;
 450                 ret = kernctl_add_callsite(fd, &callsite);
 451                 if (ret) {
 452                         WARN("Failed to add callsite to ELF userspace probe.");
 453                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 454                         goto end;
 455                 }
 456                 break;
 457         }
 458         case LTTNG_USERSPACE_PROBE_LOCATION_LOOKUP_METHOD_TYPE_TRACEPOINT_SDT:
 459         {
 460                 int i;
 461                 uint64_t *offsets = NULL;
 462                 uint32_t offsets_count;
 463                 struct lttng_kernel_abi_event_callsite callsite;
 464
 465                 /*
 466                  * This call allocates the offsets buffer. This buffer must be freed
 467                  * by the caller
 468                  */
 469                 ret = extract_userspace_probe_offset_tracepoint_sdt(location,
 470                                 uid, gid, &offsets, &offsets_count);
 471                 if (ret) {
 472                         ret = LTTNG_ERR_PROBE_LOCATION_INVAL;
 473                         goto end;
 474                 }
 475                 for (i = 0; i < offsets_count; i++) {
 476                         callsite.u.uprobe.offset = offsets[i];
 477                         ret = kernctl_add_callsite(fd, &callsite);
 478                         if (ret) {
 479                                 WARN("Failed to add callsite to SDT userspace probe");
 480                                 ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 481                                 free(offsets);
 482                                 goto end;
 483                         }
 484                 }
 485                 free(offsets);
 486                 break;
 487         }
 488         default:
 489                 ret = LTTNG_ERR_PROBE_LOCATION_INVAL;
 490                 goto end;
 491         }
 492 end:
 493         return ret;
 494 }
 495
 496 /*
 497  * Extract the offsets of the instrumentation point for the different lookup
 498  * methods.
 499  */
 500 static
 501 int userspace_probe_event_add_callsites(struct lttng_event *ev,
 502                         struct ltt_kernel_session *session, int fd)
 503 {
 504         int ret;
 505         const struct lttng_userspace_probe_location *location = NULL;
 506
 507         LTTNG_ASSERT(ev);
 508         LTTNG_ASSERT(ev->type == LTTNG_EVENT_USERSPACE_PROBE);
 509
 510         location = lttng_event_get_userspace_probe_location(ev);
 511         if (!location) {
 512                 ret = -1;
 513                 goto end;
 514         }
 515
 516         ret = userspace_probe_add_callsite(location, session->uid, session->gid,
 517                 fd);
 518         if (ret) {
 519                 WARN("Failed to add callsite to userspace probe event '%s'",
 520                                 ev->name);
 521         }
 522
 523 end:
 524         return ret;
 525 }
 526
 527 /*
 528  * Extract the offsets of the instrumentation point for the different look-up
 529  * methods.
 530  */
 531 static int userspace_probe_event_rule_add_callsites(
 532                 const struct lttng_event_rule *rule,
 533                 const struct lttng_credentials *creds,
 534                 int fd)
 535 {
 536         int ret;
 537         enum lttng_event_rule_status status;
 538         enum lttng_event_rule_type event_rule_type;
 539         const struct lttng_userspace_probe_location *location = NULL;
 540
 541         LTTNG_ASSERT(rule);
 542         LTTNG_ASSERT(creds);
 543
 544         event_rule_type = lttng_event_rule_get_type(rule);
 545         LTTNG_ASSERT(event_rule_type == LTTNG_EVENT_RULE_TYPE_KERNEL_UPROBE);
 546
 547         status = lttng_event_rule_kernel_uprobe_get_location(rule, &location);
 548         if (status != LTTNG_EVENT_RULE_STATUS_OK || !location) {
 549                 ret = -1;
 550                 goto end;
 551         }
 552
 553         ret = userspace_probe_add_callsite(location,
 554                         lttng_credentials_get_uid(creds),
 555                         lttng_credentials_get_gid(creds), fd);
 556         if (ret) {
 557                 WARN("Failed to add callsite to user space probe object: fd = %d",
 558                                 fd);
 559         }
 560
 561 end:
 562         return ret;
 563 }
 564
 565 /*
 566  * Create a kernel event, enable it to the kernel tracer and add it to the
 567  * channel event list of the kernel session.
 568  * We own filter_expression and filter.
 569  */
 570 int kernel_create_event(struct lttng_event *ev,
 571                 struct ltt_kernel_channel *channel,
 572                 char *filter_expression,
 573                 struct lttng_bytecode *filter)
 574 {
 575         int err, fd;
 576         enum lttng_error_code ret;
 577         struct ltt_kernel_event *event;
 578
 579         LTTNG_ASSERT(ev);
 580         LTTNG_ASSERT(channel);
 581
 582         /* We pass ownership of filter_expression and filter */
 583         ret = trace_kernel_create_event(ev, filter_expression,
 584                         filter, &event);
 585         if (ret != LTTNG_OK) {
 586                 goto error;
 587         }
 588
 589         fd = kernctl_create_event(channel->fd, event->event);
 590         if (fd < 0) {
 591                 switch (-fd) {
 592                 case EEXIST:
 593                         ret = LTTNG_ERR_KERN_EVENT_EXIST;
 594                         break;
 595                 case ENOSYS:
 596                         WARN("Event type not implemented");
 597                         ret = LTTNG_ERR_KERN_EVENT_ENOSYS;
 598                         break;
 599                 case ENOENT:
 600                         WARN("Event %s not found!", ev->name);
 601                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 602                         break;
 603                 default:
 604                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 605                         PERROR("create event ioctl");
 606                 }
 607                 goto free_event;
 608         }
 609
 610         event->type = ev->type;
 611         event->fd = fd;
 612         /* Prevent fd duplication after execlp() */
 613         err = fcntl(event->fd, F_SETFD, FD_CLOEXEC);
 614         if (err < 0) {
 615                 PERROR("fcntl session fd");
 616         }
 617
 618         if (filter) {
 619                 err = kernctl_filter(event->fd, filter);
 620                 if (err < 0) {
 621                         switch (-err) {
 622                         case ENOMEM:
 623                                 ret = LTTNG_ERR_FILTER_NOMEM;
 624                                 break;
 625                         default:
 626                                 ret = LTTNG_ERR_FILTER_INVAL;
 627                                 break;
 628                         }
 629                         goto filter_error;
 630                 }
 631         }
 632
 633         if (ev->type == LTTNG_EVENT_USERSPACE_PROBE) {
 634                 ret = (lttng_error_code) userspace_probe_event_add_callsites(ev, channel->session,
 635                                 event->fd);
 636                 if (ret) {
 637                         goto add_callsite_error;
 638                 }
 639         }
 640
 641         err = kernctl_enable(event->fd);
 642         if (err < 0) {
 643                 switch (-err) {
 644                 case EEXIST:
 645                         ret = LTTNG_ERR_KERN_EVENT_EXIST;
 646                         break;
 647                 default:
 648                         PERROR("enable kernel event");
 649                         ret = LTTNG_ERR_KERN_ENABLE_FAIL;
 650                         break;
 651                 }
 652                 goto enable_error;
 653         }
 654
 655         /* Add event to event list */
 656         cds_list_add(&event->list, &channel->events_list.head);
 657         channel->event_count++;
 658
 659         DBG("Event %s created (fd: %d)", ev->name, event->fd);
 660
 661         return 0;
 662
 663 add_callsite_error:
 664 enable_error:
 665 filter_error:
 666         {
 667                 int closeret;
 668
 669                 closeret = close(event->fd);
 670                 if (closeret) {
 671                         PERROR("close event fd");
 672                 }
 673         }
 674 free_event:
 675         free(event);
 676 error:
 677         return ret;
 678 }
 679
 680 /*
 681  * Disable a kernel channel.
 682  */
 683 int kernel_disable_channel(struct ltt_kernel_channel *chan)
 684 {
 685         int ret;
 686
 687         LTTNG_ASSERT(chan);
 688
 689         ret = kernctl_disable(chan->fd);
 690         if (ret < 0) {
 691                 PERROR("disable chan ioctl");
 692                 goto error;
 693         }
 694
 695         chan->enabled = 0;
 696         DBG("Kernel channel %s disabled (fd: %d, key: %" PRIu64 ")",
 697                         chan->channel->name, chan->fd, chan->key);
 698
 699         return 0;
 700
 701 error:
 702         return ret;
 703 }
 704
 705 /*
 706  * Enable a kernel channel.
 707  */
 708 int kernel_enable_channel(struct ltt_kernel_channel *chan)
 709 {
 710         int ret;
 711
 712         LTTNG_ASSERT(chan);
 713
 714         ret = kernctl_enable(chan->fd);
 715         if (ret < 0 && ret != -EEXIST) {
 716                 PERROR("Enable kernel chan");
 717                 goto error;
 718         }
 719
 720         chan->enabled = 1;
 721         DBG("Kernel channel %s enabled (fd: %d, key: %" PRIu64 ")",
 722                         chan->channel->name, chan->fd, chan->key);
 723
 724         return 0;
 725
 726 error:
 727         return ret;
 728 }
 729
 730 /*
 731  * Enable a kernel event.
 732  */
 733 int kernel_enable_event(struct ltt_kernel_event *event)
 734 {
 735         int ret;
 736
 737         LTTNG_ASSERT(event);
 738
 739         ret = kernctl_enable(event->fd);
 740         if (ret < 0) {
 741                 switch (-ret) {
 742                 case EEXIST:
 743                         ret = LTTNG_ERR_KERN_EVENT_EXIST;
 744                         break;
 745                 default:
 746                         PERROR("enable kernel event");
 747                         break;
 748                 }
 749                 goto error;
 750         }
 751
 752         event->enabled = 1;
 753         DBG("Kernel event %s enabled (fd: %d)", event->event->name, event->fd);
 754
 755         return 0;
 756
 757 error:
 758         return ret;
 759 }
 760
 761 /*
 762  * Disable a kernel event.
 763  */
 764 int kernel_disable_event(struct ltt_kernel_event *event)
 765 {
 766         int ret;
 767
 768         LTTNG_ASSERT(event);
 769
 770         ret = kernctl_disable(event->fd);
 771         if (ret < 0) {
 772                 PERROR("Failed to disable kernel event: name = '%s', fd = %d",
 773                                 event->event->name, event->fd);
 774                 goto error;
 775         }
 776
 777         event->enabled = 0;
 778         DBG("Kernel event %s disabled (fd: %d)", event->event->name, event->fd);
 779
 780         return 0;
 781
 782 error:
 783         return ret;
 784 }
 785
 786 /*
 787  * Disable a kernel event notifier.
 788  */
 789 static
 790 int kernel_disable_event_notifier_rule(struct ltt_kernel_event_notifier_rule *event)
 791 {
 792         int ret;
 793
 794         LTTNG_ASSERT(event);
 795
 796         rcu_read_lock();
 797         cds_lfht_del(kernel_token_to_event_notifier_rule_ht, &event->ht_node);
 798         rcu_read_unlock();
 799
 800         ret = kernctl_disable(event->fd);
 801         if (ret < 0) {
 802                 PERROR("Failed to disable kernel event notifier: fd = %d, token = %" PRIu64,
 803                                 event->fd, event->token);
 804                 goto error;
 805         }
 806
 807         event->enabled = 0;
 808         DBG("Disabled kernel event notifier: fd = %d, token = %" PRIu64,
 809                         event->fd, event->token);
 810
 811 error:
 812         return ret;
 813 }
 814
 815 static
 816 struct process_attr_tracker *_kernel_get_process_attr_tracker(
 817                 struct ltt_kernel_session *session,
 818                 enum lttng_process_attr process_attr)
 819 {
 820         switch (process_attr) {
 821         case LTTNG_PROCESS_ATTR_PROCESS_ID:
 822                 return session->tracker_pid;
 823         case LTTNG_PROCESS_ATTR_VIRTUAL_PROCESS_ID:
 824                 return session->tracker_vpid;
 825         case LTTNG_PROCESS_ATTR_USER_ID:
 826                 return session->tracker_uid;
 827         case LTTNG_PROCESS_ATTR_VIRTUAL_USER_ID:
 828                 return session->tracker_vuid;
 829         case LTTNG_PROCESS_ATTR_GROUP_ID:
 830                 return session->tracker_gid;
 831         case LTTNG_PROCESS_ATTR_VIRTUAL_GROUP_ID:
 832                 return session->tracker_vgid;
 833         default:
 834                 return NULL;
 835         }
 836 }
 837
 838 const struct process_attr_tracker *kernel_get_process_attr_tracker(
 839                 struct ltt_kernel_session *session,
 840                 enum lttng_process_attr process_attr)
 841 {
 842         return (const struct process_attr_tracker *)
 843                         _kernel_get_process_attr_tracker(session, process_attr);
 844 }
 845
 846 enum lttng_error_code kernel_process_attr_tracker_set_tracking_policy(
 847                 struct ltt_kernel_session *session,
 848                 enum lttng_process_attr process_attr,
 849                 enum lttng_tracking_policy policy)
 850 {
 851         int ret;
 852         enum lttng_error_code ret_code = LTTNG_OK;
 853         struct process_attr_tracker *tracker =
 854                         _kernel_get_process_attr_tracker(session, process_attr);
 855         enum lttng_tracking_policy previous_policy;
 856
 857         if (!tracker) {
 858                 ret_code = LTTNG_ERR_INVALID;
 859                 goto end;
 860         }
 861
 862         previous_policy = process_attr_tracker_get_tracking_policy(tracker);
 863         ret = process_attr_tracker_set_tracking_policy(tracker, policy);
 864         if (ret) {
 865                 ret_code = LTTNG_ERR_UNK;
 866                 goto end;
 867         }
 868
 869         if (previous_policy == policy) {
 870                 goto end;
 871         }
 872
 873         switch (policy) {
 874         case LTTNG_TRACKING_POLICY_INCLUDE_ALL:
 875                 if (process_attr == LTTNG_PROCESS_ATTR_PROCESS_ID) {
 876                         /*
 877                          * Maintain a special case for the process ID process
 878                          * attribute tracker as it was the only supported
 879                          * attribute prior to 2.12.
 880                          */
 881                         ret = kernctl_track_pid(session->fd, -1);
 882                 } else {
 883                         ret = kernctl_track_id(session->fd, process_attr, -1);
 884                 }
 885                 break;
 886         case LTTNG_TRACKING_POLICY_EXCLUDE_ALL:
 887         case LTTNG_TRACKING_POLICY_INCLUDE_SET:
 888                 /* fall-through. */
 889                 if (process_attr == LTTNG_PROCESS_ATTR_PROCESS_ID) {
 890                         /*
 891                          * Maintain a special case for the process ID process
 892                          * attribute tracker as it was the only supported
 893                          * attribute prior to 2.12.
 894                          */
 895                         ret = kernctl_untrack_pid(session->fd, -1);
 896                 } else {
 897                         ret = kernctl_untrack_id(session->fd, process_attr, -1);
 898                 }
 899                 break;
 900         default:
 901                 abort();
 902         }
 903         /* kern-ctl error handling */
 904         switch (-ret) {
 905         case 0:
 906                 ret_code = LTTNG_OK;
 907                 break;
 908         case EINVAL:
 909                 ret_code = LTTNG_ERR_INVALID;
 910                 break;
 911         case ENOMEM:
 912                 ret_code = LTTNG_ERR_NOMEM;
 913                 break;
 914         case EEXIST:
 915                 ret_code = LTTNG_ERR_PROCESS_ATTR_EXISTS;
 916                 break;
 917         default:
 918                 ret_code = LTTNG_ERR_UNK;
 919                 break;
 920         }
 921 end:
 922         return ret_code;
 923 }
 924
 925 enum lttng_error_code kernel_process_attr_tracker_inclusion_set_add_value(
 926                 struct ltt_kernel_session *session,
 927                 enum lttng_process_attr process_attr,
 928                 const struct process_attr_value *value)
 929 {
 930         int ret, integral_value;
 931         enum lttng_error_code ret_code;
 932         struct process_attr_tracker *tracker;
 933         enum process_attr_tracker_status status;
 934
 935         /*
 936          * Convert process attribute tracker value to the integral
 937          * representation required by the kern-ctl API.
 938          */
 939         switch (process_attr) {
 940         case LTTNG_PROCESS_ATTR_PROCESS_ID:
 941         case LTTNG_PROCESS_ATTR_VIRTUAL_PROCESS_ID:
 942                 integral_value = (int) value->value.pid;
 943                 break;
 944         case LTTNG_PROCESS_ATTR_USER_ID:
 945         case LTTNG_PROCESS_ATTR_VIRTUAL_USER_ID:
 946                 if (value->type == LTTNG_PROCESS_ATTR_VALUE_TYPE_USER_NAME) {
 947                         uid_t uid;
 948
 949                         ret_code = utils_user_id_from_name(
 950                                         value->value.user_name, &uid);
 951                         if (ret_code != LTTNG_OK) {
 952                                 goto end;
 953                         }
 954                         integral_value = (int) uid;
 955                 } else {
 956                         integral_value = (int) value->value.uid;
 957                 }
 958                 break;
 959         case LTTNG_PROCESS_ATTR_GROUP_ID:
 960         case LTTNG_PROCESS_ATTR_VIRTUAL_GROUP_ID:
 961                 if (value->type == LTTNG_PROCESS_ATTR_VALUE_TYPE_GROUP_NAME) {
 962                         gid_t gid;
 963
 964                         ret_code = utils_group_id_from_name(
 965                                         value->value.group_name, &gid);
 966                         if (ret_code != LTTNG_OK) {
 967                                 goto end;
 968                         }
 969                         integral_value = (int) gid;
 970                 } else {
 971                         integral_value = (int) value->value.gid;
 972                 }
 973                 break;
 974         default:
 975                 ret_code = LTTNG_ERR_INVALID;
 976                 goto end;
 977         }
 978
 979         tracker = _kernel_get_process_attr_tracker(session, process_attr);
 980         if (!tracker) {
 981                 ret_code = LTTNG_ERR_INVALID;
 982                 goto end;
 983         }
 984
 985         status = process_attr_tracker_inclusion_set_add_value(tracker, value);
 986         if (status != PROCESS_ATTR_TRACKER_STATUS_OK) {
 987                 switch (status) {
 988                 case PROCESS_ATTR_TRACKER_STATUS_EXISTS:
 989                         ret_code = LTTNG_ERR_PROCESS_ATTR_EXISTS;
 990                         break;
 991                 case PROCESS_ATTR_TRACKER_STATUS_INVALID_TRACKING_POLICY:
 992                         ret_code = LTTNG_ERR_PROCESS_ATTR_TRACKER_INVALID_TRACKING_POLICY;
 993                         break;
 994                 case PROCESS_ATTR_TRACKER_STATUS_ERROR:
 995                 default:
 996                         ret_code = LTTNG_ERR_UNK;
 997                         break;
 998                 }
 999                 goto end;
1000         }
1001
1002         DBG("Kernel track %s %d for session id %" PRIu64,
1003                         lttng_process_attr_to_string(process_attr),
1004                         integral_value, session->id);
1005         if (process_attr == LTTNG_PROCESS_ATTR_PROCESS_ID) {
1006                 /*
1007                  * Maintain a special case for the process ID process attribute
1008                  * tracker as it was the only supported attribute prior to 2.12.
1009                  */
1010                 ret = kernctl_track_pid(session->fd, integral_value);
1011         } else {
1012                 ret = kernctl_track_id(
1013                                 session->fd, process_attr, integral_value);
1014         }
1015         if (ret == 0) {
1016                 ret_code = LTTNG_OK;
1017                 goto end;
1018         }
1019
1020         kernel_wait_quiescent();
1021
1022         /* kern-ctl error handling */
1023         switch (-ret) {
1024         case 0:
1025                 ret_code = LTTNG_OK;
1026                 break;
1027         case EINVAL:
1028                 ret_code = LTTNG_ERR_INVALID;
1029                 break;
1030         case ENOMEM:
1031                 ret_code = LTTNG_ERR_NOMEM;
1032                 break;
1033         case EEXIST:
1034                 ret_code = LTTNG_ERR_PROCESS_ATTR_EXISTS;
1035                 break;
1036         default:
1037                 ret_code = LTTNG_ERR_UNK;
1038                 break;
1039         }
1040
1041         /* Attempt to remove the value from the tracker. */
1042         status = process_attr_tracker_inclusion_set_remove_value(
1043                         tracker, value);
1044         if (status != PROCESS_ATTR_TRACKER_STATUS_OK) {
1045                 ERR("Failed to roll-back the tracking of kernel %s process attribute %d while handling a kern-ctl error",
1046                                 lttng_process_attr_to_string(process_attr),
1047                                 integral_value);
1048         }
1049 end:
1050         return ret_code;
1051 }
1052
1053 enum lttng_error_code kernel_process_attr_tracker_inclusion_set_remove_value(
1054                 struct ltt_kernel_session *session,
1055                 enum lttng_process_attr process_attr,
1056                 const struct process_attr_value *value)
1057 {
1058         int ret, integral_value;
1059         enum lttng_error_code ret_code;
1060         struct process_attr_tracker *tracker;
1061         enum process_attr_tracker_status status;
1062
1063         /*
1064          * Convert process attribute tracker value to the integral
1065          * representation required by the kern-ctl API.
1066          */
1067         switch (process_attr) {
1068         case LTTNG_PROCESS_ATTR_PROCESS_ID:
1069         case LTTNG_PROCESS_ATTR_VIRTUAL_PROCESS_ID:
1070                 integral_value = (int) value->value.pid;
1071                 break;
1072         case LTTNG_PROCESS_ATTR_USER_ID:
1073         case LTTNG_PROCESS_ATTR_VIRTUAL_USER_ID:
1074                 if (value->type == LTTNG_PROCESS_ATTR_VALUE_TYPE_USER_NAME) {
1075                         uid_t uid;
1076
1077                         ret_code = utils_user_id_from_name(
1078                                         value->value.user_name, &uid);
1079                         if (ret_code != LTTNG_OK) {
1080                                 goto end;
1081                         }
1082                         integral_value = (int) uid;
1083                 } else {
1084                         integral_value = (int) value->value.uid;
1085                 }
1086                 break;
1087         case LTTNG_PROCESS_ATTR_GROUP_ID:
1088         case LTTNG_PROCESS_ATTR_VIRTUAL_GROUP_ID:
1089                 if (value->type == LTTNG_PROCESS_ATTR_VALUE_TYPE_GROUP_NAME) {
1090                         gid_t gid;
1091
1092                         ret_code = utils_group_id_from_name(
1093                                         value->value.group_name, &gid);
1094                         if (ret_code != LTTNG_OK) {
1095                                 goto end;
1096                         }
1097                         integral_value = (int) gid;
1098                 } else {
1099                         integral_value = (int) value->value.gid;
1100                 }
1101                 break;
1102         default:
1103                 ret_code = LTTNG_ERR_INVALID;
1104                 goto end;
1105         }
1106
1107         tracker = _kernel_get_process_attr_tracker(session, process_attr);
1108         if (!tracker) {
1109                 ret_code = LTTNG_ERR_INVALID;
1110                 goto end;
1111         }
1112
1113         status = process_attr_tracker_inclusion_set_remove_value(
1114                         tracker, value);
1115         if (status != PROCESS_ATTR_TRACKER_STATUS_OK) {
1116                 switch (status) {
1117                 case PROCESS_ATTR_TRACKER_STATUS_MISSING:
1118                         ret_code = LTTNG_ERR_PROCESS_ATTR_MISSING;
1119                         break;
1120                 case PROCESS_ATTR_TRACKER_STATUS_INVALID_TRACKING_POLICY:
1121                         ret_code = LTTNG_ERR_PROCESS_ATTR_TRACKER_INVALID_TRACKING_POLICY;
1122                         break;
1123                 case PROCESS_ATTR_TRACKER_STATUS_ERROR:
1124                 default:
1125                         ret_code = LTTNG_ERR_UNK;
1126                         break;
1127                 }
1128                 goto end;
1129         }
1130
1131         DBG("Kernel track %s %d for session id %" PRIu64,
1132                         lttng_process_attr_to_string(process_attr),
1133                         integral_value, session->id);
1134         if (process_attr == LTTNG_PROCESS_ATTR_PROCESS_ID) {
1135                 /*
1136                  * Maintain a special case for the process ID process attribute
1137                  * tracker as it was the only supported attribute prior to 2.12.
1138                  */
1139                 ret = kernctl_untrack_pid(session->fd, integral_value);
1140         } else {
1141                 ret = kernctl_untrack_id(
1142                                 session->fd, process_attr, integral_value);
1143         }
1144         if (ret == 0) {
1145                 ret_code = LTTNG_OK;
1146                 goto end;
1147         }
1148         kernel_wait_quiescent();
1149
1150         /* kern-ctl error handling */
1151         switch (-ret) {
1152         case 0:
1153                 ret_code = LTTNG_OK;
1154                 break;
1155         case EINVAL:
1156                 ret_code = LTTNG_ERR_INVALID;
1157                 break;
1158         case ENOMEM:
1159                 ret_code = LTTNG_ERR_NOMEM;
1160                 break;
1161         case ENOENT:
1162                 ret_code = LTTNG_ERR_PROCESS_ATTR_MISSING;
1163                 break;
1164         default:
1165                 ret_code = LTTNG_ERR_UNK;
1166                 break;
1167         }
1168
1169         /* Attempt to add the value to the tracker. */
1170         status = process_attr_tracker_inclusion_set_add_value(
1171                         tracker, value);
1172         if (status != PROCESS_ATTR_TRACKER_STATUS_OK) {
1173                 ERR("Failed to roll-back the tracking of kernel %s process attribute %d while handling a kern-ctl error",
1174                                 lttng_process_attr_to_string(process_attr),
1175                                 integral_value);
1176         }
1177 end:
1178         return ret_code;
1179 }
1180
1181 /*
1182  * Create kernel metadata, open from the kernel tracer and add it to the
1183  * kernel session.
1184  */
1185 int kernel_open_metadata(struct ltt_kernel_session *session)
1186 {
1187         int ret;
1188         struct ltt_kernel_metadata *lkm = NULL;
1189
1190         LTTNG_ASSERT(session);
1191
1192         /* Allocate kernel metadata */
1193         lkm = trace_kernel_create_metadata();
1194         if (lkm == NULL) {
1195                 goto error;
1196         }
1197
1198         /* Kernel tracer metadata creation */
1199         ret = kernctl_open_metadata(session->fd, &lkm->conf->attr);
1200         if (ret < 0) {
1201                 goto error_open;
1202         }
1203
1204         lkm->fd = ret;
1205         lkm->key = ++next_kernel_channel_key;
1206         /* Prevent fd duplication after execlp() */
1207         ret = fcntl(lkm->fd, F_SETFD, FD_CLOEXEC);
1208         if (ret < 0) {
1209                 PERROR("fcntl session fd");
1210         }
1211
1212         session->metadata = lkm;
1213
1214         DBG("Kernel metadata opened (fd: %d)", lkm->fd);
1215
1216         return 0;
1217
1218 error_open:
1219         trace_kernel_destroy_metadata(lkm);
1220 error:
1221         return -1;
1222 }
1223
1224 /*
1225  * Start tracing session.
1226  */
1227 int kernel_start_session(struct ltt_kernel_session *session)
1228 {
1229         int ret;
1230
1231         LTTNG_ASSERT(session);
1232
1233         ret = kernctl_start_session(session->fd);
1234         if (ret < 0) {
1235                 PERROR("ioctl start session");
1236                 goto error;
1237         }
1238
1239         DBG("Kernel session started");
1240
1241         return 0;
1242
1243 error:
1244         return ret;
1245 }
1246
1247 /*
1248  * Make a kernel wait to make sure in-flight probe have completed.
1249  */
1250 void kernel_wait_quiescent(void)
1251 {
1252         int ret;
1253         int fd = kernel_tracer_fd;
1254
1255         DBG("Kernel quiescent wait on %d", fd);
1256
1257         ret = kernctl_wait_quiescent(fd);
1258         if (ret < 0) {
1259                 PERROR("wait quiescent ioctl");
1260                 ERR("Kernel quiescent wait failed");
1261         }
1262 }
1263
1264 /*
1265  *  Force flush buffer of metadata.
1266  */
1267 int kernel_metadata_flush_buffer(int fd)
1268 {
1269         int ret;
1270
1271         DBG("Kernel flushing metadata buffer on fd %d", fd);
1272
1273         ret = kernctl_buffer_flush(fd);
1274         if (ret < 0) {
1275                 ERR("Fail to flush metadata buffers %d (ret: %d)", fd, ret);
1276         }
1277
1278         return 0;
1279 }
1280
1281 /*
1282  * Force flush buffer for channel.
1283  */
1284 int kernel_flush_buffer(struct ltt_kernel_channel *channel)
1285 {
1286         int ret;
1287         struct ltt_kernel_stream *stream;
1288
1289         LTTNG_ASSERT(channel);
1290
1291         DBG("Flush buffer for channel %s", channel->channel->name);
1292
1293         cds_list_for_each_entry(stream, &channel->stream_list.head, list) {
1294                 DBG("Flushing channel stream %d", stream->fd);
1295                 ret = kernctl_buffer_flush(stream->fd);
1296                 if (ret < 0) {
1297                         PERROR("ioctl");
1298                         ERR("Fail to flush buffer for stream %d (ret: %d)",
1299                                         stream->fd, ret);
1300                 }
1301         }
1302
1303         return 0;
1304 }
1305
1306 /*
1307  * Stop tracing session.
1308  */
1309 int kernel_stop_session(struct ltt_kernel_session *session)
1310 {
1311         int ret;
1312
1313         LTTNG_ASSERT(session);
1314
1315         ret = kernctl_stop_session(session->fd);
1316         if (ret < 0) {
1317                 goto error;
1318         }
1319
1320         DBG("Kernel session stopped");
1321
1322         return 0;
1323
1324 error:
1325         return ret;
1326 }
1327
1328 /*
1329  * Open stream of channel, register it to the kernel tracer and add it
1330  * to the stream list of the channel.
1331  *
1332  * Note: given that the streams may appear in random order wrt CPU
1333  * number (e.g. cpu hotplug), the index value of the stream number in
1334  * the stream name is not necessarily linked to the CPU number.
1335  *
1336  * Return the number of created stream. Else, a negative value.
1337  */
1338 int kernel_open_channel_stream(struct ltt_kernel_channel *channel)
1339 {
1340         int ret;
1341         struct ltt_kernel_stream *lks;
1342
1343         LTTNG_ASSERT(channel);
1344
1345         while ((ret = kernctl_create_stream(channel->fd)) >= 0) {
1346                 lks = trace_kernel_create_stream(channel->channel->name,
1347                                 channel->stream_count);
1348                 if (lks == NULL) {
1349                         ret = close(ret);
1350                         if (ret) {
1351                                 PERROR("close");
1352                         }
1353                         goto error;
1354                 }
1355
1356                 lks->fd = ret;
1357                 /* Prevent fd duplication after execlp() */
1358                 ret = fcntl(lks->fd, F_SETFD, FD_CLOEXEC);
1359                 if (ret < 0) {
1360                         PERROR("fcntl session fd");
1361                 }
1362
1363                 lks->tracefile_size = channel->channel->attr.tracefile_size;
1364                 lks->tracefile_count = channel->channel->attr.tracefile_count;
1365
1366                 /* Add stream to channel stream list */
1367                 cds_list_add(&lks->list, &channel->stream_list.head);
1368                 channel->stream_count++;
1369
1370                 DBG("Kernel stream %s created (fd: %d, state: %d)", lks->name, lks->fd,
1371                                 lks->state);
1372         }
1373
1374         return channel->stream_count;
1375
1376 error:
1377         return -1;
1378 }
1379
1380 /*
1381  * Open the metadata stream and set it to the kernel session.
1382  */
1383 int kernel_open_metadata_stream(struct ltt_kernel_session *session)
1384 {
1385         int ret;
1386
1387         LTTNG_ASSERT(session);
1388
1389         ret = kernctl_create_stream(session->metadata->fd);
1390         if (ret < 0) {
1391                 PERROR("kernel create metadata stream");
1392                 goto error;
1393         }
1394
1395         DBG("Kernel metadata stream created (fd: %d)", ret);
1396         session->metadata_stream_fd = ret;
1397         /* Prevent fd duplication after execlp() */
1398         ret = fcntl(session->metadata_stream_fd, F_SETFD, FD_CLOEXEC);
1399         if (ret < 0) {
1400                 PERROR("fcntl session fd");
1401         }
1402
1403         return 0;
1404
1405 error:
1406         return -1;
1407 }
1408
1409 /*
1410  * Get the event list from the kernel tracer and return the number of elements.
1411  */
1412 ssize_t kernel_list_events(struct lttng_event **events)
1413 {
1414         int fd, ret;
1415         char *event;
1416         size_t nbmem, count = 0;
1417         FILE *fp;
1418         struct lttng_event *elist;
1419
1420         LTTNG_ASSERT(events);
1421
1422         fd = kernctl_tracepoint_list(kernel_tracer_fd);
1423         if (fd < 0) {
1424                 PERROR("kernel tracepoint list");
1425                 goto error;
1426         }
1427
1428         fp = fdopen(fd, "r");
1429         if (fp == NULL) {
1430                 PERROR("kernel tracepoint list fdopen");
1431                 goto error_fp;
1432         }
1433
1434         /*
1435          * Init memory size counter
1436          * See kernel-ctl.h for explanation of this value
1437          */
1438         nbmem = KERNEL_EVENT_INIT_LIST_SIZE;
1439         elist = calloc<lttng_event>(nbmem);
1440         if (elist == NULL) {
1441                 PERROR("alloc list events");
1442                 count = -ENOMEM;
1443                 goto end;
1444         }
1445
1446         while (fscanf(fp, "event { name = %m[^;]; };\n", &event) == 1) {
1447                 if (count >= nbmem) {
1448                         struct lttng_event *new_elist;
1449                         size_t new_nbmem;
1450
1451                         new_nbmem = nbmem << 1;
1452                         DBG("Reallocating event list from %zu to %zu bytes",
1453                                         nbmem, new_nbmem);
1454                         new_elist = (lttng_event *) realloc(elist, new_nbmem * sizeof(struct lttng_event));
1455                         if (new_elist == NULL) {
1456                                 PERROR("realloc list events");
1457                                 free(event);
1458                                 free(elist);
1459                                 count = -ENOMEM;
1460                                 goto end;
1461                         }
1462                         /* Zero the new memory */
1463                         memset(new_elist + nbmem, 0,
1464                                 (new_nbmem - nbmem) * sizeof(struct lttng_event));
1465                         nbmem = new_nbmem;
1466                         elist = new_elist;
1467                 }
1468                 strncpy(elist[count].name, event, LTTNG_SYMBOL_NAME_LEN);
1469                 elist[count].name[LTTNG_SYMBOL_NAME_LEN - 1] = '\0';
1470                 elist[count].enabled = -1;
1471                 count++;
1472                 free(event);
1473         }
1474
1475         *events = elist;
1476         DBG("Kernel list events done (%zu events)", count);
1477 end:
1478         ret = fclose(fp);       /* closes both fp and fd */
1479         if (ret) {
1480                 PERROR("fclose");
1481         }
1482         return count;
1483
1484 error_fp:
1485         ret = close(fd);
1486         if (ret) {
1487                 PERROR("close");
1488         }
1489 error:
1490         return -1;
1491 }
1492
1493 /*
1494  * Get kernel version and validate it.
1495  */
1496 int kernel_validate_version(struct lttng_kernel_abi_tracer_version *version,
1497                 struct lttng_kernel_abi_tracer_abi_version *abi_version)
1498 {
1499         int ret;
1500
1501         ret = kernctl_tracer_version(kernel_tracer_fd, version);
1502         if (ret < 0) {
1503                 ERR("Failed to retrieve the lttng-modules version");
1504                 goto error;
1505         }
1506
1507         /* Validate version */
1508         if (version->major != VERSION_MAJOR) {
1509                 ERR("Kernel tracer major version (%d) is not compatible with lttng-tools major version (%d)",
1510                         version->major, VERSION_MAJOR);
1511                 goto error_version;
1512         }
1513         ret = kernctl_tracer_abi_version(kernel_tracer_fd, abi_version);
1514         if (ret < 0) {
1515                 ERR("Failed to retrieve lttng-modules ABI version");
1516                 goto error;
1517         }
1518         if (abi_version->major != LTTNG_KERNEL_ABI_MAJOR_VERSION) {
1519                 ERR("Kernel tracer ABI version (%d.%d) does not match the expected ABI major version (%d.*)",
1520                         abi_version->major, abi_version->minor,
1521                         LTTNG_KERNEL_ABI_MAJOR_VERSION);
1522                 goto error;
1523         }
1524         DBG2("Kernel tracer version validated (%d.%d, ABI %d.%d)",
1525                         version->major, version->minor,
1526                         abi_version->major, abi_version->minor);
1527         return 0;
1528
1529 error_version:
1530         ret = -1;
1531
1532 error:
1533         ERR("Kernel tracer version check failed; kernel tracing will not be available");
1534         return ret;
1535 }
1536
1537 /*
1538  * Kernel work-arounds called at the start of sessiond main().
1539  */
1540 int init_kernel_workarounds(void)
1541 {
1542         int ret;
1543         FILE *fp;
1544
1545         /*
1546          * boot_id needs to be read once before being used concurrently
1547          * to deal with a Linux kernel race. A fix is proposed for
1548          * upstream, but the work-around is needed for older kernels.
1549          */
1550         fp = fopen("/proc/sys/kernel/random/boot_id", "r");
1551         if (!fp) {
1552                 goto end_boot_id;
1553         }
1554         while (!feof(fp)) {
1555                 char buf[37] = "";
1556
1557                 ret = fread(buf, 1, sizeof(buf), fp);
1558                 if (ret < 0) {
1559                         /* Ignore error, we don't really care */
1560                 }
1561         }
1562         ret = fclose(fp);
1563         if (ret) {
1564                 PERROR("fclose");
1565         }
1566 end_boot_id:
1567         return 0;
1568 }
1569
1570 /*
1571  * Teardown of a kernel session, keeping data required by destroy notifiers.
1572  */
1573 void kernel_destroy_session(struct ltt_kernel_session *ksess)
1574 {
1575         struct lttng_trace_chunk *trace_chunk;
1576
1577         if (ksess == NULL) {
1578                 DBG3("No kernel session when tearing down session");
1579                 return;
1580         }
1581
1582         DBG("Tearing down kernel session");
1583         trace_chunk = ksess->current_trace_chunk;
1584
1585         /*
1586          * Destroy channels on the consumer if at least one FD has been sent and we
1587          * are in no output mode because the streams are in *no* monitor mode so we
1588          * have to send a command to clean them up or else they leaked.
1589          */
1590         if (!ksess->output_traces && ksess->consumer_fds_sent) {
1591                 int ret;
1592                 struct consumer_socket *socket;
1593                 struct lttng_ht_iter iter;
1594
1595                 /* For each consumer socket. */
1596                 rcu_read_lock();
1597                 cds_lfht_for_each_entry(ksess->consumer->socks->ht, &iter.iter,
1598                                 socket, node.node) {
1599                         struct ltt_kernel_channel *chan;
1600
1601                         /* For each channel, ask the consumer to destroy it. */
1602                         cds_list_for_each_entry(chan, &ksess->channel_list.head, list) {
1603                                 ret = kernel_consumer_destroy_channel(socket, chan);
1604                                 if (ret < 0) {
1605                                         /* Consumer is probably dead. Use next socket. */
1606                                         continue;
1607                                 }
1608                         }
1609                 }
1610                 rcu_read_unlock();
1611         }
1612
1613         /* Close any relayd session */
1614         consumer_output_send_destroy_relayd(ksess->consumer);
1615
1616         trace_kernel_destroy_session(ksess);
1617         lttng_trace_chunk_put(trace_chunk);
1618 }
1619
1620 /* Teardown of data required by destroy notifiers. */
1621 void kernel_free_session(struct ltt_kernel_session *ksess)
1622 {
1623         if (ksess == NULL) {
1624                 return;
1625         }
1626         trace_kernel_free_session(ksess);
1627 }
1628
1629 /*
1630  * Destroy a kernel channel object. It does not do anything on the tracer side.
1631  */
1632 void kernel_destroy_channel(struct ltt_kernel_channel *kchan)
1633 {
1634         struct ltt_kernel_session *ksess = NULL;
1635
1636         LTTNG_ASSERT(kchan);
1637         LTTNG_ASSERT(kchan->channel);
1638
1639         DBG3("Kernel destroy channel %s", kchan->channel->name);
1640
1641         /* Update channel count of associated session. */
1642         if (kchan->session) {
1643                 /* Keep pointer reference so we can update it after the destroy. */
1644                 ksess = kchan->session;
1645         }
1646
1647         trace_kernel_destroy_channel(kchan);
1648
1649         /*
1650          * At this point the kernel channel is not visible anymore. This is safe
1651          * since in order to work on a visible kernel session, the tracing session
1652          * lock (ltt_session.lock) MUST be acquired.
1653          */
1654         if (ksess) {
1655                 ksess->channel_count--;
1656         }
1657 }
1658
1659 /*
1660  * Take a snapshot for a given kernel session.
1661  *
1662  * Return LTTNG_OK on success or else return a LTTNG_ERR code.
1663  */
1664 enum lttng_error_code kernel_snapshot_record(
1665                 struct ltt_kernel_session *ksess,
1666                 const struct consumer_output *output,
1667                 uint64_t nb_packets_per_stream)
1668 {
1669         int err, ret, saved_metadata_fd;
1670         enum lttng_error_code status = LTTNG_OK;
1671         struct consumer_socket *socket;
1672         struct lttng_ht_iter iter;
1673         struct ltt_kernel_metadata *saved_metadata;
1674         char *trace_path = NULL;
1675         size_t consumer_path_offset = 0;
1676
1677         LTTNG_ASSERT(ksess);
1678         LTTNG_ASSERT(ksess->consumer);
1679         LTTNG_ASSERT(output);
1680
1681         DBG("Kernel snapshot record started");
1682
1683         /* Save current metadata since the following calls will change it. */
1684         saved_metadata = ksess->metadata;
1685         saved_metadata_fd = ksess->metadata_stream_fd;
1686
1687         rcu_read_lock();
1688
1689         ret = kernel_open_metadata(ksess);
1690         if (ret < 0) {
1691                 status = LTTNG_ERR_KERN_META_FAIL;
1692                 goto error;
1693         }
1694
1695         ret = kernel_open_metadata_stream(ksess);
1696         if (ret < 0) {
1697                 status = LTTNG_ERR_KERN_META_FAIL;
1698                 goto error_open_stream;
1699         }
1700
1701         trace_path = setup_channel_trace_path(ksess->consumer,
1702                         "", &consumer_path_offset);
1703         if (!trace_path) {
1704                 status = LTTNG_ERR_INVALID;
1705                 goto error;
1706         }
1707         /* Send metadata to consumer and snapshot everything. */
1708         cds_lfht_for_each_entry(output->socks->ht, &iter.iter,
1709                         socket, node.node) {
1710                 struct ltt_kernel_channel *chan;
1711
1712                 pthread_mutex_lock(socket->lock);
1713                 /* This stream must not be monitored by the consumer. */
1714                 ret = kernel_consumer_add_metadata(socket, ksess, 0);
1715                 pthread_mutex_unlock(socket->lock);
1716                 if (ret < 0) {
1717                         status = LTTNG_ERR_KERN_META_FAIL;
1718                         goto error_consumer;
1719                 }
1720
1721                 /* For each channel, ask the consumer to snapshot it. */
1722                 cds_list_for_each_entry(chan, &ksess->channel_list.head, list) {
1723                         status = consumer_snapshot_channel(socket, chan->key, output, 0,
1724                                         &trace_path[consumer_path_offset],
1725                                         nb_packets_per_stream);
1726                         if (status != LTTNG_OK) {
1727                                 (void) kernel_consumer_destroy_metadata(socket,
1728                                                 ksess->metadata);
1729                                 goto error_consumer;
1730                         }
1731                 }
1732
1733                 /* Snapshot metadata, */
1734                 status = consumer_snapshot_channel(socket, ksess->metadata->key, output,
1735                                 1, &trace_path[consumer_path_offset], 0);
1736                 if (status != LTTNG_OK) {
1737                         goto error_consumer;
1738                 }
1739
1740                 /*
1741                  * The metadata snapshot is done, ask the consumer to destroy it since
1742                  * it's not monitored on the consumer side.
1743                  */
1744                 (void) kernel_consumer_destroy_metadata(socket, ksess->metadata);
1745         }
1746
1747 error_consumer:
1748         /* Close newly opened metadata stream. It's now on the consumer side. */
1749         err = close(ksess->metadata_stream_fd);
1750         if (err < 0) {
1751                 PERROR("close snapshot kernel");
1752         }
1753
1754 error_open_stream:
1755         trace_kernel_destroy_metadata(ksess->metadata);
1756 error:
1757         /* Restore metadata state.*/
1758         ksess->metadata = saved_metadata;
1759         ksess->metadata_stream_fd = saved_metadata_fd;
1760         rcu_read_unlock();
1761         free(trace_path);
1762         return status;
1763 }
1764
1765 /*
1766  * Get the syscall mask array from the kernel tracer.
1767  *
1768  * Return 0 on success else a negative value. In both case, syscall_mask should
1769  * be freed.
1770  */
1771 int kernel_syscall_mask(int chan_fd, char **syscall_mask, uint32_t *nr_bits)
1772 {
1773         LTTNG_ASSERT(syscall_mask);
1774         LTTNG_ASSERT(nr_bits);
1775
1776         return kernctl_syscall_mask(chan_fd, syscall_mask, nr_bits);
1777 }
1778
1779 static
1780 int kernel_tracer_abi_greater_or_equal(unsigned int major, unsigned int minor)
1781 {
1782         int ret;
1783         struct lttng_kernel_abi_tracer_abi_version abi;
1784
1785         ret = kernctl_tracer_abi_version(kernel_tracer_fd, &abi);
1786         if (ret < 0) {
1787                 ERR("Failed to retrieve lttng-modules ABI version");
1788                 goto error;
1789         }
1790
1791         ret = abi.major > major || (abi.major == major && abi.minor >= minor);
1792 error:
1793         return ret;
1794 }
1795
1796 /*
1797  * Check for the support of the RING_BUFFER_SNAPSHOT_SAMPLE_POSITIONS via abi
1798  * version number.
1799  *
1800  * Return 1 on success, 0 when feature is not supported, negative value in case
1801  * of errors.
1802  */
1803 int kernel_supports_ring_buffer_snapshot_sample_positions(void)
1804 {
1805         /*
1806          * RING_BUFFER_SNAPSHOT_SAMPLE_POSITIONS was introduced in 2.3
1807          */
1808         return kernel_tracer_abi_greater_or_equal(2, 3);
1809 }
1810
1811 /*
1812  * Check for the support of the packet sequence number via abi version number.
1813  *
1814  * Return 1 on success, 0 when feature is not supported, negative value in case
1815  * of errors.
1816  */
1817 int kernel_supports_ring_buffer_packet_sequence_number(void)
1818 {
1819         /*
1820          * Packet sequence number was introduced in LTTng 2.8,
1821          * lttng-modules ABI 2.1.
1822          */
1823         return kernel_tracer_abi_greater_or_equal(2, 1);
1824 }
1825
1826 /*
1827  * Check for the support of event notifiers via abi version number.
1828  *
1829  * Return 1 on success, 0 when feature is not supported, negative value in case
1830  * of errors.
1831  */
1832 int kernel_supports_event_notifiers(void)
1833 {
1834         /*
1835          * Event notifiers were introduced in LTTng 2.13, lttng-modules ABI 2.6.
1836          */
1837         return kernel_tracer_abi_greater_or_equal(2, 6);
1838 }
1839
1840 /*
1841  * Rotate a kernel session.
1842  *
1843  * Return LTTNG_OK on success or else an LTTng error code.
1844  */
1845 enum lttng_error_code kernel_rotate_session(struct ltt_session *session)
1846 {
1847         int ret;
1848         enum lttng_error_code status = LTTNG_OK;
1849         struct consumer_socket *socket;
1850         struct lttng_ht_iter iter;
1851         struct ltt_kernel_session *ksess = session->kernel_session;
1852
1853         LTTNG_ASSERT(ksess);
1854         LTTNG_ASSERT(ksess->consumer);
1855
1856         DBG("Rotate kernel session %s started (session %" PRIu64 ")",
1857                         session->name, session->id);
1858
1859         rcu_read_lock();
1860
1861         /*
1862          * Note that this loop will end after one iteration given that there is
1863          * only one kernel consumer.
1864          */
1865         cds_lfht_for_each_entry(ksess->consumer->socks->ht, &iter.iter,
1866                         socket, node.node) {
1867                 struct ltt_kernel_channel *chan;
1868
1869                 /* For each channel, ask the consumer to rotate it. */
1870                 cds_list_for_each_entry(chan, &ksess->channel_list.head, list) {
1871                         DBG("Rotate kernel channel %" PRIu64 ", session %s",
1872                                         chan->key, session->name);
1873                         ret = consumer_rotate_channel(socket, chan->key,
1874                                         ksess->consumer,
1875                                         /* is_metadata_channel */ false);
1876                         if (ret < 0) {
1877                                 status = LTTNG_ERR_ROTATION_FAIL_CONSUMER;
1878                                 goto error;
1879                         }
1880                 }
1881
1882                 /*
1883                  * Rotate the metadata channel.
1884                  */
1885                 ret = consumer_rotate_channel(socket, ksess->metadata->key,
1886                                 ksess->consumer,
1887                                 /* is_metadata_channel */ true);
1888                 if (ret < 0) {
1889                         status = LTTNG_ERR_ROTATION_FAIL_CONSUMER;
1890                         goto error;
1891                 }
1892         }
1893
1894 error:
1895         rcu_read_unlock();
1896         return status;
1897 }
1898
1899 enum lttng_error_code kernel_create_channel_subdirectories(
1900                 const struct ltt_kernel_session *ksess)
1901 {
1902         enum lttng_error_code ret = LTTNG_OK;
1903         enum lttng_trace_chunk_status chunk_status;
1904
1905         rcu_read_lock();
1906         LTTNG_ASSERT(ksess->current_trace_chunk);
1907
1908         /*
1909          * Create the index subdirectory which will take care
1910          * of implicitly creating the channel's path.
1911          */
1912         chunk_status = lttng_trace_chunk_create_subdirectory(
1913                         ksess->current_trace_chunk,
1914                         DEFAULT_KERNEL_TRACE_DIR "/" DEFAULT_INDEX_DIR);
1915         if (chunk_status != LTTNG_TRACE_CHUNK_STATUS_OK) {
1916                 ret = LTTNG_ERR_CREATE_DIR_FAIL;
1917                 goto error;
1918         }
1919 error:
1920         rcu_read_unlock();
1921         return ret;
1922 }
1923
1924 /*
1925  * Setup necessary data for kernel tracer action.
1926  */
1927 int init_kernel_tracer(void)
1928 {
1929         int ret;
1930         bool is_root = !getuid();
1931
1932         /* Modprobe lttng kernel modules */
1933         ret = modprobe_lttng_control();
1934         if (ret < 0) {
1935                 goto error;
1936         }
1937
1938         /* Open debugfs lttng */
1939         kernel_tracer_fd = open(module_proc_lttng, O_RDWR);
1940         if (kernel_tracer_fd < 0) {
1941                 DBG("Failed to open %s", module_proc_lttng);
1942                 goto error_open;
1943         }
1944
1945         /* Validate kernel version */
1946         ret = kernel_validate_version(&the_kernel_tracer_version,
1947                         &the_kernel_tracer_abi_version);
1948         if (ret < 0) {
1949                 goto error_version;
1950         }
1951
1952         ret = modprobe_lttng_data();
1953         if (ret < 0) {
1954                 goto error_modules;
1955         }
1956
1957         ret = kernel_supports_ring_buffer_snapshot_sample_positions();
1958         if (ret < 0) {
1959                 goto error_modules;
1960         }
1961         if (ret < 1) {
1962                 WARN("Kernel tracer does not support buffer monitoring. "
1963                         "The monitoring timer of channels in the kernel domain "
1964                         "will be set to 0 (disabled).");
1965         }
1966
1967         ret = kernel_supports_event_notifiers();
1968         if (ret < 0) {
1969                 ERR("Failed to check for kernel tracer event notifier support");
1970                 goto error_modules;
1971         }
1972         ret = kernel_create_event_notifier_group(&kernel_tracer_event_notifier_group_fd);
1973         if (ret < 0) {
1974                 /* This is not fatal. */
1975                 WARN("Failed to create kernel event notifier group");
1976                 kernel_tracer_event_notifier_group_fd = -1;
1977         } else {
1978                 enum event_notifier_error_accounting_status error_accounting_status;
1979                 enum lttng_error_code error_code_ret =
1980                                 kernel_create_event_notifier_group_notification_fd(
1981                                                 &kernel_tracer_event_notifier_group_notification_fd);
1982
1983                 if (error_code_ret != LTTNG_OK) {
1984                         goto error_modules;
1985                 }
1986
1987                 error_accounting_status = event_notifier_error_accounting_register_kernel(
1988                                 kernel_tracer_event_notifier_group_fd);
1989                 if (error_accounting_status != EVENT_NOTIFIER_ERROR_ACCOUNTING_STATUS_OK) {
1990                         ERR("Failed to initialize event notifier error accounting for kernel tracer");
1991                         error_code_ret = LTTNG_ERR_EVENT_NOTIFIER_ERROR_ACCOUNTING;
1992                         goto error_modules;
1993                 }
1994
1995                 kernel_token_to_event_notifier_rule_ht = cds_lfht_new(
1996                                 DEFAULT_HT_SIZE, 1, 0,
1997                                 CDS_LFHT_AUTO_RESIZE | CDS_LFHT_ACCOUNTING,
1998                                 NULL);
1999                 if (!kernel_token_to_event_notifier_rule_ht) {
2000                         goto error_token_ht;
2001                 }
2002         }
2003
2004         DBG("Kernel tracer initialized: kernel tracer fd = %d, event notifier group fd = %d, event notifier group notification fd = %d",
2005                         kernel_tracer_fd, kernel_tracer_event_notifier_group_fd,
2006                         kernel_tracer_event_notifier_group_notification_fd);
2007
2008         ret = syscall_init_table(kernel_tracer_fd);
2009         if (ret < 0) {
2010                 ERR("Unable to populate syscall table. Syscall tracing won't "
2011                         "work for this session daemon.");
2012         }
2013
2014         return 0;
2015
2016 error_version:
2017         modprobe_remove_lttng_control();
2018         ret = close(kernel_tracer_fd);
2019         if (ret) {
2020                 PERROR("Failed to close kernel tracer file descriptor: fd = %d",
2021                                 kernel_tracer_fd);
2022         }
2023
2024         kernel_tracer_fd = -1;
2025         return LTTNG_ERR_KERN_VERSION;
2026
2027
2028 error_token_ht:
2029         ret = close(kernel_tracer_event_notifier_group_notification_fd);
2030         if (ret) {
2031                 PERROR("Failed to close kernel tracer event notifier group notification file descriptor: fd = %d",
2032                                 kernel_tracer_event_notifier_group_notification_fd);
2033         }
2034
2035         kernel_tracer_event_notifier_group_notification_fd = -1;
2036
2037 error_modules:
2038         ret = close(kernel_tracer_event_notifier_group_fd);
2039         if (ret) {
2040                 PERROR("Failed to close kernel tracer event notifier group file descriptor: fd = %d",
2041                                 kernel_tracer_event_notifier_group_fd);
2042         }
2043
2044         kernel_tracer_event_notifier_group_fd = -1;
2045
2046         ret = close(kernel_tracer_fd);
2047         if (ret) {
2048                 PERROR("Failed to close kernel tracer file descriptor: fd = %d",
2049                                 kernel_tracer_fd);
2050         }
2051
2052         kernel_tracer_fd = -1;
2053
2054 error_open:
2055         modprobe_remove_lttng_control();
2056
2057 error:
2058         WARN("No kernel tracer available");
2059         kernel_tracer_fd = -1;
2060         if (!is_root) {
2061                 return LTTNG_ERR_NEED_ROOT_SESSIOND;
2062         } else {
2063                 return LTTNG_ERR_KERN_NA;
2064         }
2065 }
2066
2067 void cleanup_kernel_tracer(void)
2068 {
2069         DBG2("Closing kernel event notifier group notification file descriptor");
2070         if (kernel_tracer_event_notifier_group_notification_fd >= 0) {
2071                 int ret = notification_thread_command_remove_tracer_event_source(
2072                                 the_notification_thread_handle,
2073                                 kernel_tracer_event_notifier_group_notification_fd);
2074                 if (ret != LTTNG_OK) {
2075                         ERR("Failed to remove kernel event notifier notification from notification thread");
2076                 }
2077
2078                 ret = close(kernel_tracer_event_notifier_group_notification_fd);
2079                 if (ret) {
2080                         PERROR("Failed to close kernel event notifier group notification file descriptor: fd = %d",
2081                                         kernel_tracer_event_notifier_group_notification_fd);
2082                 }
2083
2084                 kernel_tracer_event_notifier_group_notification_fd = -1;
2085         }
2086
2087         if (kernel_token_to_event_notifier_rule_ht) {
2088                 const int ret = cds_lfht_destroy(
2089                                 kernel_token_to_event_notifier_rule_ht, NULL);
2090                 LTTNG_ASSERT(ret == 0);
2091         }
2092
2093         DBG2("Closing kernel event notifier group file descriptor");
2094         if (kernel_tracer_event_notifier_group_fd >= 0) {
2095                 const int ret = close(kernel_tracer_event_notifier_group_fd);
2096
2097                 if (ret) {
2098                         PERROR("Failed to close kernel event notifier group file descriptor: fd = %d",
2099                                         kernel_tracer_event_notifier_group_fd);
2100                 }
2101
2102                 kernel_tracer_event_notifier_group_fd = -1;
2103         }
2104
2105         DBG2("Closing kernel fd");
2106         if (kernel_tracer_fd >= 0) {
2107                 const int ret = close(kernel_tracer_fd);
2108
2109                 if (ret) {
2110                         PERROR("Failed to close kernel tracer file descriptor: fd = %d",
2111                                         kernel_tracer_fd);
2112                 }
2113
2114                 kernel_tracer_fd = -1;
2115         }
2116
2117         free(syscall_table);
2118 }
2119
2120 bool kernel_tracer_is_initialized(void)
2121 {
2122         return kernel_tracer_fd >= 0;
2123 }
2124
2125 /*
2126  *  Clear a kernel session.
2127  *
2128  * Return LTTNG_OK on success or else an LTTng error code.
2129  */
2130 enum lttng_error_code kernel_clear_session(struct ltt_session *session)
2131 {
2132         int ret;
2133         enum lttng_error_code status = LTTNG_OK;
2134         struct consumer_socket *socket;
2135         struct lttng_ht_iter iter;
2136         struct ltt_kernel_session *ksess = session->kernel_session;
2137
2138         LTTNG_ASSERT(ksess);
2139         LTTNG_ASSERT(ksess->consumer);
2140
2141         DBG("Clear kernel session %s (session %" PRIu64 ")",
2142                         session->name, session->id);
2143
2144         rcu_read_lock();
2145
2146         if (ksess->active) {
2147                 ERR("Expecting inactive session %s (%" PRIu64 ")", session->name, session->id);
2148                 status = LTTNG_ERR_FATAL;
2149                 goto end;
2150         }
2151
2152         /*
2153          * Note that this loop will end after one iteration given that there is
2154          * only one kernel consumer.
2155          */
2156         cds_lfht_for_each_entry(ksess->consumer->socks->ht, &iter.iter,
2157                         socket, node.node) {
2158                 struct ltt_kernel_channel *chan;
2159
2160                 /* For each channel, ask the consumer to clear it. */
2161                 cds_list_for_each_entry(chan, &ksess->channel_list.head, list) {
2162                         DBG("Clear kernel channel %" PRIu64 ", session %s",
2163                                         chan->key, session->name);
2164                         ret = consumer_clear_channel(socket, chan->key);
2165                         if (ret < 0) {
2166                                 goto error;
2167                         }
2168                 }
2169
2170                 if (!ksess->metadata) {
2171                         /*
2172                          * Nothing to do for the metadata.
2173                          * This is a snapshot session.
2174                          * The metadata is genererated on the fly.
2175                          */
2176                         continue;
2177                 }
2178
2179                 /*
2180                  * Clear the metadata channel.
2181                  * Metadata channel is not cleared per se but we still need to
2182                  * perform a rotation operation on it behind the scene.
2183                  */
2184                 ret = consumer_clear_channel(socket, ksess->metadata->key);
2185                 if (ret < 0) {
2186                         goto error;
2187                 }
2188         }
2189
2190         goto end;
2191 error:
2192         switch (-ret) {
2193         case LTTCOMM_CONSUMERD_RELAYD_CLEAR_DISALLOWED:
2194               status = LTTNG_ERR_CLEAR_RELAY_DISALLOWED;
2195               break;
2196         default:
2197               status = LTTNG_ERR_CLEAR_FAIL_CONSUMER;
2198               break;
2199         }
2200 end:
2201         rcu_read_unlock();
2202         return status;
2203 }
2204
2205 enum lttng_error_code kernel_create_event_notifier_group_notification_fd(
2206                 int *event_notifier_group_notification_fd)
2207 {
2208         int local_fd = -1, ret;
2209         enum lttng_error_code error_code_ret;
2210
2211         LTTNG_ASSERT(event_notifier_group_notification_fd);
2212
2213         ret = kernctl_create_event_notifier_group_notification_fd(
2214                         kernel_tracer_event_notifier_group_fd);
2215         if (ret < 0) {
2216                 PERROR("Failed to create kernel event notifier group notification file descriptor");
2217                 error_code_ret = LTTNG_ERR_EVENT_NOTIFIER_GROUP_NOTIFICATION_FD;
2218                 goto error;
2219         }
2220
2221         local_fd = ret;
2222
2223         /* Prevent fd duplication after execlp(). */
2224         ret = fcntl(local_fd, F_SETFD, FD_CLOEXEC);
2225         if (ret < 0) {
2226                 PERROR("Failed to set FD_CLOEXEC on kernel event notifier group notification file descriptor: fd = %d",
2227                                 local_fd);
2228                 error_code_ret = LTTNG_ERR_EVENT_NOTIFIER_GROUP_NOTIFICATION_FD;
2229                 goto error;
2230         }
2231
2232         DBG("Created kernel notifier group notification file descriptor: fd = %d",
2233                         local_fd);
2234         error_code_ret = LTTNG_OK;
2235         *event_notifier_group_notification_fd = local_fd;
2236         local_fd = -1;
2237
2238 error:
2239         if (local_fd >= 0) {
2240                 ret = close(local_fd);
2241                 if (ret) {
2242                         PERROR("Failed to close kernel event notifier group notification file descriptor: fd = %d",
2243                                         local_fd);
2244                 }
2245         }
2246
2247         return error_code_ret;
2248 }
2249
2250 enum lttng_error_code kernel_destroy_event_notifier_group_notification_fd(
2251                 int event_notifier_group_notification_fd)
2252 {
2253         enum lttng_error_code ret_code = LTTNG_OK;
2254
2255         DBG("Closing event notifier group notification file descriptor: fd = %d",
2256                         event_notifier_group_notification_fd);
2257         if (event_notifier_group_notification_fd >= 0) {
2258                 const int ret = close(event_notifier_group_notification_fd);
2259                 if (ret) {
2260                         PERROR("Failed to close event notifier group notification file descriptor: fd = %d",
2261                                         event_notifier_group_notification_fd);
2262                 }
2263         }
2264
2265         return ret_code;
2266 }
2267
2268 static
2269 unsigned long hash_trigger(const struct lttng_trigger *trigger)
2270 {
2271         const struct lttng_condition *condition =
2272                         lttng_trigger_get_const_condition(trigger);
2273
2274         return lttng_condition_hash(condition);
2275 }
2276
2277 static
2278 int match_trigger(struct cds_lfht_node *node, const void *key)
2279 {
2280         const struct ltt_kernel_event_notifier_rule *event_notifier_rule;
2281         const struct lttng_trigger *trigger = (lttng_trigger *) key;
2282
2283         event_notifier_rule = caa_container_of(node,
2284                         const struct ltt_kernel_event_notifier_rule, ht_node);
2285
2286         return lttng_trigger_is_equal(trigger, event_notifier_rule->trigger);
2287 }
2288
2289 static enum lttng_error_code kernel_create_event_notifier_rule(
2290                 struct lttng_trigger *trigger,
2291                 const struct lttng_credentials *creds, uint64_t token)
2292 {
2293         int err, fd, ret = 0;
2294         enum lttng_error_code error_code_ret;
2295         enum lttng_condition_status condition_status;
2296         enum lttng_condition_type condition_type;
2297         enum lttng_event_rule_type event_rule_type;
2298         struct ltt_kernel_event_notifier_rule *event_notifier_rule;
2299         struct lttng_kernel_abi_event_notifier kernel_event_notifier = {};
2300         unsigned int capture_bytecode_count = 0, i;
2301         const struct lttng_condition *condition = NULL;
2302         const struct lttng_event_rule *event_rule = NULL;
2303         enum lttng_condition_status cond_status;
2304
2305         LTTNG_ASSERT(trigger);
2306
2307         condition = lttng_trigger_get_const_condition(trigger);
2308         LTTNG_ASSERT(condition);
2309
2310         condition_type = lttng_condition_get_type(condition);
2311         LTTNG_ASSERT(condition_type == LTTNG_CONDITION_TYPE_EVENT_RULE_MATCHES);
2312
2313         /* Does not acquire a reference. */
2314         condition_status = lttng_condition_event_rule_matches_get_rule(
2315                         condition, &event_rule);
2316         LTTNG_ASSERT(condition_status == LTTNG_CONDITION_STATUS_OK);
2317         LTTNG_ASSERT(event_rule);
2318
2319         event_rule_type = lttng_event_rule_get_type(event_rule);
2320         LTTNG_ASSERT(event_rule_type != LTTNG_EVENT_RULE_TYPE_UNKNOWN);
2321
2322         error_code_ret = trace_kernel_create_event_notifier_rule(trigger, token,
2323                         lttng_condition_event_rule_matches_get_error_counter_index(
2324                                         condition),
2325                         &event_notifier_rule);
2326         if (error_code_ret != LTTNG_OK) {
2327                 goto error;
2328         }
2329
2330         error_code_ret = trace_kernel_init_event_notifier_from_event_rule(
2331                         event_rule, &kernel_event_notifier);
2332         if (error_code_ret != LTTNG_OK) {
2333                 goto free_event;
2334         }
2335
2336         kernel_event_notifier.event.token = event_notifier_rule->token;
2337         kernel_event_notifier.error_counter_idx =
2338                         lttng_condition_event_rule_matches_get_error_counter_index(
2339                                         condition);
2340
2341         fd = kernctl_create_event_notifier(
2342                         kernel_tracer_event_notifier_group_fd,
2343                         &kernel_event_notifier);
2344         if (fd < 0) {
2345                 switch (-fd) {
2346                 case EEXIST:
2347                         error_code_ret = LTTNG_ERR_KERN_EVENT_EXIST;
2348                         break;
2349                 case ENOSYS:
2350                         WARN("Failed to create kernel event notifier: not notifier type not implemented");
2351                         error_code_ret = LTTNG_ERR_KERN_EVENT_ENOSYS;
2352                         break;
2353                 case ENOENT:
2354                         WARN("Failed to create kernel event notifier: not found: name = '%s'",
2355                                         kernel_event_notifier.event.name);
2356                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2357                         break;
2358                 default:
2359                         PERROR("Failed to create kernel event notifier: error code = %d, name = '%s'",
2360                                         fd, kernel_event_notifier.event.name);
2361                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2362                 }
2363                 goto free_event;
2364         }
2365
2366         event_notifier_rule->fd = fd;
2367         /* Prevent fd duplication after execlp(). */
2368         err = fcntl(event_notifier_rule->fd, F_SETFD, FD_CLOEXEC);
2369         if (err < 0) {
2370                 PERROR("Failed to set FD_CLOEXEC on kernel event notifier file descriptor: fd = %d",
2371                                 fd);
2372                 error_code_ret = LTTNG_ERR_FATAL;
2373                 goto set_cloexec_error;
2374         }
2375
2376         if (event_notifier_rule->filter) {
2377                 err = kernctl_filter(event_notifier_rule->fd, event_notifier_rule->filter);
2378                 if (err < 0) {
2379                         switch (-err) {
2380                         case ENOMEM:
2381                                 error_code_ret = LTTNG_ERR_FILTER_NOMEM;
2382                                 break;
2383                         default:
2384                                 error_code_ret = LTTNG_ERR_FILTER_INVAL;
2385                                 break;
2386                         }
2387                         goto filter_error;
2388                 }
2389         }
2390
2391         if (lttng_event_rule_get_type(event_rule) ==
2392                         LTTNG_EVENT_RULE_TYPE_KERNEL_UPROBE) {
2393                 ret = userspace_probe_event_rule_add_callsites(
2394                                 event_rule, creds, event_notifier_rule->fd);
2395                 if (ret) {
2396                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2397                         goto add_callsite_error;
2398                 }
2399         }
2400
2401         /* Set the capture bytecode if any. */
2402         cond_status = lttng_condition_event_rule_matches_get_capture_descriptor_count(
2403                         condition, &capture_bytecode_count);
2404         LTTNG_ASSERT(cond_status == LTTNG_CONDITION_STATUS_OK);
2405
2406         for (i = 0; i < capture_bytecode_count; i++) {
2407                 const struct lttng_bytecode *capture_bytecode =
2408                                 lttng_condition_event_rule_matches_get_capture_bytecode_at_index(
2409                                                 condition, i);
2410
2411                 if (capture_bytecode == NULL) {
2412                         ERR("Unexpected NULL capture bytecode on condition");
2413                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2414                         goto capture_error;
2415                 }
2416
2417                 ret = kernctl_capture(event_notifier_rule->fd, capture_bytecode);
2418                 if (ret < 0) {
2419                         ERR("Failed to set capture bytecode on event notifier rule fd: fd = %d",
2420                                         event_notifier_rule->fd);
2421                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2422                         goto capture_error;
2423                 }
2424         }
2425
2426         err = kernctl_enable(event_notifier_rule->fd);
2427         if (err < 0) {
2428                 switch (-err) {
2429                 case EEXIST:
2430                         error_code_ret = LTTNG_ERR_KERN_EVENT_EXIST;
2431                         break;
2432                 default:
2433                         PERROR("enable kernel event notifier");
2434                         error_code_ret = LTTNG_ERR_KERN_ENABLE_FAIL;
2435                         break;
2436                 }
2437                 goto enable_error;
2438         }
2439
2440         /* Add trigger to kernel token mapping in the hash table. */
2441         rcu_read_lock();
2442         cds_lfht_add(kernel_token_to_event_notifier_rule_ht, hash_trigger(trigger),
2443                         &event_notifier_rule->ht_node);
2444         rcu_read_unlock();
2445
2446         DBG("Created kernel event notifier: name = '%s', fd = %d",
2447                         kernel_event_notifier.event.name,
2448                         event_notifier_rule->fd);
2449
2450         return LTTNG_OK;
2451
2452 capture_error:
2453 add_callsite_error:
2454 enable_error:
2455 set_cloexec_error:
2456 filter_error:
2457         {
2458                 const int close_ret = close(event_notifier_rule->fd);
2459
2460                 if (close_ret) {
2461                         PERROR("Failed to close kernel event notifier file descriptor: fd = %d",
2462                                         event_notifier_rule->fd);
2463                 }
2464         }
2465 free_event:
2466         free(event_notifier_rule);
2467 error:
2468         return error_code_ret;
2469 }
2470
2471 enum lttng_error_code kernel_register_event_notifier(
2472                 struct lttng_trigger *trigger,
2473                 const struct lttng_credentials *cmd_creds)
2474 {
2475         enum lttng_error_code ret;
2476         enum lttng_condition_status status;
2477         enum lttng_domain_type domain_type;
2478         const struct lttng_event_rule *event_rule;
2479         const struct lttng_condition *const condition =
2480                         lttng_trigger_get_const_condition(trigger);
2481         const uint64_t token = lttng_trigger_get_tracer_token(trigger);
2482
2483         LTTNG_ASSERT(condition);
2484
2485         /* Does not acquire a reference to the event rule. */
2486         status = lttng_condition_event_rule_matches_get_rule(
2487                         condition, &event_rule);
2488         LTTNG_ASSERT(status == LTTNG_CONDITION_STATUS_OK);
2489
2490         domain_type = lttng_event_rule_get_domain_type(event_rule);
2491         LTTNG_ASSERT(domain_type == LTTNG_DOMAIN_KERNEL);
2492
2493         ret = kernel_create_event_notifier_rule(trigger, cmd_creds, token);
2494         if (ret != LTTNG_OK) {
2495                 ERR("Failed to create kernel event notifier rule");
2496         }
2497
2498         return ret;
2499 }
2500
2501 enum lttng_error_code kernel_unregister_event_notifier(
2502                 const struct lttng_trigger *trigger)
2503 {
2504         struct ltt_kernel_event_notifier_rule *token_event_rule_element;
2505         struct cds_lfht_node *node;
2506         struct cds_lfht_iter iter;
2507         enum lttng_error_code error_code_ret;
2508         int ret;
2509
2510         rcu_read_lock();
2511
2512         cds_lfht_lookup(kernel_token_to_event_notifier_rule_ht,
2513                         hash_trigger(trigger), match_trigger, trigger, &iter);
2514
2515         node = cds_lfht_iter_get_node(&iter);
2516         if (!node) {
2517                 error_code_ret = LTTNG_ERR_TRIGGER_NOT_FOUND;
2518                 goto error;
2519         }
2520
2521         token_event_rule_element = caa_container_of(node,
2522                         struct ltt_kernel_event_notifier_rule, ht_node);
2523
2524         ret = kernel_disable_event_notifier_rule(token_event_rule_element);
2525         if (ret) {
2526                 error_code_ret = LTTNG_ERR_FATAL;
2527                 goto error;
2528         }
2529
2530         trace_kernel_destroy_event_notifier_rule(token_event_rule_element);
2531         error_code_ret = LTTNG_OK;
2532
2533 error:
2534         rcu_read_unlock();
2535
2536         return error_code_ret;
2537 }
2538
2539 int kernel_get_notification_fd(void)
2540 {
2541         return kernel_tracer_event_notifier_group_notification_fd;
2542 }