From 57b90af7b1977684094706818e387433f50b7d48 Mon Sep 17 00:00:00 2001 From: =?utf8?q?J=C3=A9r=C3=A9mie=20Galarneau?= Date: Tue, 3 Jan 2023 18:41:23 -0500 Subject: [PATCH] Fix: sessiond: instance uuid is not sufficiently unique MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Observed issue ============== Tracing a cluster of machines -- all launched simultaneously -- to the same relay daemon occasionally produces corrupted traces. The size of packets received (as seen from the relay daemon logs) and that of those present in the on-disk stream occasionally didn't match. The traces were observed to all relate to the same trace UUID, but present packet begin timestamps that were not monotonic for a given stream. This causes both Babeltrace 1.x and 2.x to fail to open the traces with different error messages related to clocks. Cause ===== On start, the session daemon generates a UUID to uniquely identify the sessiond instance. Since the UUID generation utils use time() to seed the random number generator, two session daemons launched within the same second can end up with the same instance UUID. Since the relay daemon relies on this UUID to uniquely identify a session daemon accross its various connections, identifier clashes can cause streams from the same `uid` or `pid` to become scrambled resulting in corrupted traces. Solution ======== The UUID utils now initializes its random seed using the getrandom() API in non-blocking mode. If that fails -- most likely because the random pool is depleted or the syscall is not available on the platform -- it falls back to using a hash of two time readings (with nanosecond precision), of the hostname, and the PID. Known drawbacks =============== This fix implements many fallbacks, each with their own caveats and we don't have full test coverage for all of those for the moment. This article presents the different drawbacks of using /dev/urandom vs getrandom(). https://lwn.net/Articles/884875/ As for the pseudo-random time and configuration based fallback, it is meant as a last resort for platforms or configurations where both getrandom() (old kernels or non-Linux platforms) and /dev/urandom (e.g. locked-down container) are not be available. I haven't done a formal analysis of the entropy of this home-grown method. The practical use-case we want to enable is launching multiple virtual machines (or containers) at roughly the same time and ensure that they don't end up using the same sessiond UUID. In that respect, having a different host name and minute timing changes seem enough to prevent a UUID clash. Using the PID as part of the hash also helps when launching multiple session daemons simultaneously for different users. Change-Id: I064753b9ff0f5bf2279be0bd0cfbfd2b0dd19bfc Signed-off-by: Jérémie Galarneau --- configure.ac | 4 +- src/common/Makefile.am | 2 + src/common/file-descriptor.hpp | 66 +++++++++++ src/common/random.cpp | 196 +++++++++++++++++++++++++++++++++ src/common/random.hpp | 49 +++++++++ src/common/uuid.cpp | 19 ++-- tests/unit/test_uuid.cpp | 5 + 7 files changed, 331 insertions(+), 10 deletions(-) create mode 100644 src/common/file-descriptor.hpp create mode 100644 src/common/random.cpp create mode 100644 src/common/random.hpp diff --git a/configure.ac b/configure.ac index 6697fcd46..187590476 100644 --- a/configure.ac +++ b/configure.ac @@ -300,7 +300,7 @@ AC_CHECK_HEADERS([ \ signal.h stdlib.h sys/un.h sys/socket.h stdlib.h stdio.h \ getopt.h sys/ipc.h sys/shm.h popt.h grp.h arpa/inet.h \ netdb.h netinet/in.h paths.h stddef.h sys/file.h sys/ioctl.h \ - sys/mount.h sys/param.h sys/time.h elf.h + sys/mount.h sys/param.h sys/time.h elf.h sys/random.h sys/syscall.h ]) AM_CONDITIONAL([HAVE_ELF_H], [test x$ac_cv_header_elf_h = xyes]) @@ -312,7 +312,7 @@ AC_CHECK_FUNCS([ \ mkdir munmap putenv realpath rmdir socket strchr strcspn strdup \ strncasecmp strndup strnlen strpbrk strrchr strstr strtol strtoul \ strtoull dirfd gethostbyname2 getipnodebyname epoll_create1 \ - sched_getcpu sysconf sync_file_range + sched_getcpu sysconf sync_file_range getrandom ]) # Check for pthread_setname_np and pthread_getname_np diff --git a/src/common/Makefile.am b/src/common/Makefile.am index 522a1ee32..165b7c3a1 100644 --- a/src/common/Makefile.am +++ b/src/common/Makefile.am @@ -84,6 +84,7 @@ libcommon_lgpl_la_SOURCES = \ event-rule/jul-logging.cpp \ event-rule/python-logging.cpp \ exception.cpp exception.hpp \ + file-descriptor.hpp \ fd-handle.cpp fd-handle.hpp\ format.hpp \ kernel-probe.cpp \ @@ -98,6 +99,7 @@ libcommon_lgpl_la_SOURCES = \ payload.cpp payload.hpp \ payload-view.cpp payload-view.hpp \ pthread-lock.hpp \ + random.cpp random.hpp \ readwrite.cpp readwrite.hpp \ runas.cpp runas.hpp \ session-descriptor.cpp \ diff --git a/src/common/file-descriptor.hpp b/src/common/file-descriptor.hpp new file mode 100644 index 000000000..6354a12e2 --- /dev/null +++ b/src/common/file-descriptor.hpp @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2023 Jérémie Galarneau + * + * SPDX-License-Identifier: LGPL-2.1-only + * + */ + +#include +#include + +#include + +#include + +namespace lttng { + +/* + * RAII wrapper around a UNIX file descriptor. A file_descriptor's underlying + * file descriptor + */ +class file_descriptor { +public: + explicit file_descriptor(int raw_fd) noexcept : _raw_fd{raw_fd} + { + LTTNG_ASSERT(_is_valid_fd(_raw_fd)); + } + + file_descriptor(const file_descriptor&) = delete; + + file_descriptor(file_descriptor&& other) : _raw_fd{-1} + { + LTTNG_ASSERT(_is_valid_fd(_raw_fd)); + std::swap(_raw_fd, other._raw_fd); + } + + ~file_descriptor() + { + if (!_is_valid_fd(_raw_fd)) { + return; + } + + const auto ret = ::close(_raw_fd); + if (ret) { + PERROR("%s", + fmt::format("Failed to close file descriptor: fd = {}", + _raw_fd) + .c_str()); + } + } + + int fd() const noexcept + { + LTTNG_ASSERT(_is_valid_fd(_raw_fd)); + return _raw_fd; + } + +private: + static bool _is_valid_fd(int fd) + { + return fd >= 0; + } + + int _raw_fd; +}; + +} /* namespace lttng */ \ No newline at end of file diff --git a/src/common/random.cpp b/src/common/random.cpp new file mode 100644 index 000000000..b5669d097 --- /dev/null +++ b/src/common/random.cpp @@ -0,0 +1,196 @@ +/* + * Copyright (C) 2023 Jérémie Galarneau + * + * SPDX-License-Identifier: LGPL-2.1-only + * + */ + +#include + +#include +#include +#include +#include +#include +#include + +#include + +#ifdef HAVE_SYS_SYSCALL_H +#include +#endif + +#define LTTNG_THROW_RANDOM_PRODUCTION_ERROR(msg) \ + throw lttng::random::production_error(msg, __FILE__, __func__, __LINE__) + +namespace { +/* getrandom is available in Linux >= 3.17. */ +#if defined(__linux__) && defined(SYS_getrandom) && defined(HAVE_SYS_RANDOM_H) + +#include + +/* A glibc wrapper is provided only for glibc >= 2.25. */ +#if defined(HAVE_GETRANDOM) +/* Simply use the existing wrapper, passing the non-block flag. */ +ssize_t _call_getrandom_nonblock(char *out_data, std::size_t size) +{ + return getrandom(out_data, size, GRND_NONBLOCK); +} +#else +ssize_t _call_getrandom_nonblock(char *out_data, std::size_t size) +{ + const int grnd_nonblock_flag = 0x1; + + auto ret = syscall(SYS_getrandom, out_data, size, grnd_nonblock_flag); + if (ret < 0) { + errno = -ret; + ret = -1; + } + + return ret; +} +#endif /* defined(HAVE_GETRANDOM) */ + +/* Returns either with a full read or throws. */ +void getrandom_nonblock(char *out_data, std::size_t size) +{ + /* + * Since GRND_RANDOM is _not_ used, a partial read can only be caused + * by a signal interruption. In this case, retry. + */ + ssize_t ret; + + do { + ret = _call_getrandom_nonblock(out_data, size); + } while ((ret > 0 && ret != size) || (ret == -1 && errno == EINTR)); + + if (ret < 0) { + LTTNG_THROW_POSIX( + fmt::format("Failed to get true random data using getrandom(): size={}", + size), + errno); + } +} +#else /* defined(__linux__) && defined(SYS_getrandom) && defined(HAVE_SYS_RANDOM_H) */ +void getrandom_nonblock(char *out_data, std::size_t size) +{ + LTTNG_THROW_RANDOM_PRODUCTION_ERROR( + "getrandom() is not supported by this platform"); +} +#endif /* defined(__linux__) && defined(SYS_getrandom) && defined(HAVE_SYS_RANDOM_H) */ + +lttng::random::seed_t produce_pseudo_random_seed() +{ + int ret; + struct timespec real_time = {}; + struct timespec monotonic_time = {}; + unsigned long hash_seed; + char hostname[LTTNG_HOST_NAME_MAX] = {}; + unsigned long seed; + + ret = clock_gettime(CLOCK_REALTIME, &real_time); + if (ret) { + LTTNG_THROW_POSIX("Failed to read real time while generating pseudo-random seed", + errno); + } + + ret = clock_gettime(CLOCK_MONOTONIC, &monotonic_time); + if (ret) { + LTTNG_THROW_POSIX( + "Failed to read monotonic time while generating pseudo-random seed", + errno); + } + + ret = gethostname(hostname, sizeof(hostname)); + if (ret) { + LTTNG_THROW_POSIX("Failed to get host name while generating pseudo-random seed", + errno); + } + + hash_seed = (unsigned long) real_time.tv_nsec ^ (unsigned long) real_time.tv_sec ^ + (unsigned long) monotonic_time.tv_nsec ^ + (unsigned long) monotonic_time.tv_sec; + seed = hash_key_ulong((void *) real_time.tv_sec, hash_seed); + seed ^= hash_key_ulong((void *) real_time.tv_nsec, hash_seed); + seed ^= hash_key_ulong((void *) monotonic_time.tv_sec, hash_seed); + seed ^= hash_key_ulong((void *) monotonic_time.tv_nsec, hash_seed); + + const unsigned long pid = getpid(); + seed ^= hash_key_ulong((void *) pid, hash_seed); + seed ^= hash_key_str(hostname, hash_seed); + + return static_cast(seed); +} + +lttng::random::seed_t produce_random_seed_from_urandom() +{ + /* + * Open /dev/urandom as a file_descriptor, or throw on error. The + * lambda is used to reduce the scope of the raw fd as much as possible. + */ + lttng::file_descriptor urandom{[]() { + const auto urandom_raw_fd = open("/dev/urandom", O_RDONLY | O_CLOEXEC); + + if (urandom_raw_fd < 0) { + LTTNG_THROW_POSIX("Failed to open `/dev/urandom`", errno); + } + + return urandom_raw_fd; + }()}; + + lttng::random::seed_t seed; + const auto read_ret = lttng_read(urandom.fd(), &seed, sizeof(seed)); + if (read_ret != sizeof(seed)) { + LTTNG_THROW_POSIX(fmt::format("Failed to read from `/dev/urandom`: size={}", + sizeof(seed)), + errno); + } + + return seed; +} + +} /* namespace */ + +lttng::random::production_error::production_error(const std::string& msg, + const char *file_name, + const char *function_name, + unsigned int line_number) : + lttng::runtime_error(msg, file_name, function_name, line_number) +{ +} + +lttng::random::seed_t lttng::random::produce_true_random_seed() +{ + lttng::random::seed_t seed; + + getrandom_nonblock(reinterpret_cast(&seed), sizeof(seed)); + return seed; +} + +lttng::random::seed_t lttng::random::produce_best_effort_random_seed() +{ + try { + return lttng::random::produce_true_random_seed(); + } catch (std::exception& e) { + WARN("%s", + fmt::format("Failed to produce a random seed using getrandom(), falling back to pseudo-random device seed generation which will block until its pool is initialized: {}", + e.what()) + .c_str()); + } + + try { + /* + * Can fail for various reasons, including not being accessible + * under some containerized environments. + */ + produce_random_seed_from_urandom(); + } catch (std::exception& e) { + WARN("%s", + fmt::format("Failed to produce a random seed from the urandom device: {}", + e.what()) + .c_str()); + } + + /* Fallback to time-based seed generation. */ + return produce_pseudo_random_seed(); +} diff --git a/src/common/random.hpp b/src/common/random.hpp new file mode 100644 index 000000000..0430485c3 --- /dev/null +++ b/src/common/random.hpp @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2023 Jérémie Galarneau + * + * SPDX-License-Identifier: LGPL-2.1-only + * + */ + +#ifndef LTTNG_RANDOM_H +#define LTTNG_RANDOM_H + +#include "exception.hpp" + +#include +#include + +namespace lttng { +namespace random { + +using seed_t = unsigned int; + +class production_error : public ::lttng::runtime_error { +public: + explicit production_error(const std::string& msg, + const char *file_name, + const char *function_name, + unsigned int line_number); +}; + +/* + * Get a seed from a reliable source of randomness without blocking, raising + * an exception on failure. + */ +seed_t produce_true_random_seed(); + +/* + * Get a random seed making a best-effort to use a true randomness source, + * but falling back to a pseudo-random seed based on the time and various system + * configuration values on failure. + * + * Note that this function attempts to use the urandom device, which will block + * in the unlikely event that its pool is uninitialized, on platforms that don't + * provide getrandom(). + */ +seed_t produce_best_effort_random_seed(); + +} /* namespace random */ +} /* namespace lttng */ + +#endif /* LTTNG_RANDOM_H */ diff --git a/src/common/uuid.cpp b/src/common/uuid.cpp index dd59edc31..bef038ac6 100644 --- a/src/common/uuid.cpp +++ b/src/common/uuid.cpp @@ -7,6 +7,10 @@ */ #include +#include +#include +#include + #include #include #include @@ -78,18 +82,17 @@ int lttng_uuid_generate(lttng_uuid& uuid_out) int i, ret = 0; if (!lttng_uuid_is_init) { - /* - * We don't need cryptographic quality randomness to - * generate UUIDs, seed rand with the epoch. - */ - const time_t epoch = time(NULL); - - if (epoch == (time_t) -1) { + try { + srand(lttng::random::produce_best_effort_random_seed()); + } catch (std::exception& e) { + ERR("%s", + fmt::format("Failed to initialize random seed during generation of UUID: {}", + e.what()) + .c_str()); ret = -1; goto end; } - srand(epoch); lttng_uuid_is_init = true; } diff --git a/tests/unit/test_uuid.cpp b/tests/unit/test_uuid.cpp index 3ead2a254..6357c3adf 100644 --- a/tests/unit/test_uuid.cpp +++ b/tests/unit/test_uuid.cpp @@ -41,6 +41,11 @@ static const char invalid_str_4[] = "2d-6c6d756574-470e-9142-a4e6ad03f143"; static const char invalid_str_5[] = "4542ad19-9e4f-4931-8261-2101c3e089ae7"; static const char invalid_str_6[] = "XX0123"; +/* For error.hpp */ +int lttng_opt_quiet = 1; +int lttng_opt_verbose = 0; +int lttng_opt_mi; + static void run_test_lttng_uuid_from_str(void) { -- 2.34.1