src/lttng-context-perf-counters.c

   1 /* SPDX-License-Identifier: (GPL-2.0-only or LGPL-2.1-only)
   2  *
   3  * lttng-context-perf-counters.c
   4  *
   5  * LTTng performance monitoring counters (perf-counters) integration module.
   6  *
   7  * Copyright (C) 2009-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   8  */
   9
  10 #include <linux/module.h>
  11 #include <linux/slab.h>
  12 #include <linux/perf_event.h>
  13 #include <linux/list.h>
  14 #include <linux/string.h>
  15 #include <linux/cpu.h>
  16 #include <lttng/events.h>
  17 #include <lttng/events-internal.h>
  18 #include <ringbuffer/frontend_types.h>
  19 #include <wrapper/vmalloc.h>
  20 #include <wrapper/perf.h>
  21 #include <lttng/tracer.h>
  22
  23 static
  24 size_t perf_counter_get_size(void *priv, struct lttng_kernel_probe_ctx *probe_ctx, size_t offset)
  25 {
  26         size_t size = 0;
  27
  28         size += lib_ring_buffer_align(offset, lttng_alignof(uint64_t));
  29         size += sizeof(uint64_t);
  30         return size;
  31 }
  32
  33 static
  34 void perf_counter_record(void *priv, struct lttng_kernel_probe_ctx *probe_ctx,
  35                          struct lttng_kernel_ring_buffer_ctx *ctx,
  36                          struct lttng_kernel_channel_buffer *chan)
  37 {
  38         struct lttng_perf_counter_field *perf_field = (struct lttng_perf_counter_field *) priv;
  39         struct perf_event *event;
  40         uint64_t value;
  41
  42         event = perf_field->e[ctx->priv.reserve_cpu];
  43         if (likely(event)) {
  44                 if (unlikely(event->state == PERF_EVENT_STATE_ERROR)) {
  45                         value = 0;
  46                 } else {
  47                         event->pmu->read(event);
  48                         value = local64_read(&event->count);
  49                 }
  50         } else {
  51                 /*
  52                  * Perf chooses not to be clever and not to support enabling a
  53                  * perf counter before the cpu is brought up. Therefore, we need
  54                  * to support having events coming (e.g. scheduler events)
  55                  * before the counter is setup. Write an arbitrary 0 in this
  56                  * case.
  57                  */
  58                 value = 0;
  59         }
  60         chan->ops->event_write(ctx, &value, sizeof(value), lttng_alignof(value));
  61 }
  62
  63 #if defined(CONFIG_PERF_EVENTS) && (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(3,1,0))
  64 static
  65 void overflow_callback(struct perf_event *event,
  66                        struct perf_sample_data *data,
  67                        struct pt_regs *regs)
  68 {
  69 }
  70 #else
  71 static
  72 void overflow_callback(struct perf_event *event, int nmi,
  73                        struct perf_sample_data *data,
  74                        struct pt_regs *regs)
  75 {
  76 }
  77 #endif
  78
  79 static
  80 void lttng_destroy_perf_counter_ctx_field(void *priv)
  81 {
  82         struct lttng_perf_counter_field *perf_field = priv;
  83         struct perf_event **events = perf_field->e;
  84
  85 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,10,0))
  86         {
  87                 int ret;
  88
  89                 ret = cpuhp_state_remove_instance(lttng_hp_online,
  90                         &perf_field->cpuhp_online.node);
  91                 WARN_ON(ret);
  92                 ret = cpuhp_state_remove_instance(lttng_hp_prepare,
  93                         &perf_field->cpuhp_prepare.node);
  94                 WARN_ON(ret);
  95         }
  96 #else /* #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,10,0)) */
  97         {
  98                 int cpu;
  99
 100                 get_online_cpus();
 101                 for_each_online_cpu(cpu)
 102                         perf_event_release_kernel(events[cpu]);
 103                 put_online_cpus();
 104 #ifdef CONFIG_HOTPLUG_CPU
 105                 unregister_cpu_notifier(&perf_field->nb);
 106 #endif
 107         }
 108 #endif /* #else #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,10,0)) */
 109         kfree(perf_field->name);
 110         kfree(perf_field->attr);
 111         kfree(perf_field->event_field);
 112         lttng_kvfree(events);
 113         kfree(perf_field);
 114 }
 115
 116 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,10,0))
 117
 118 int lttng_cpuhp_perf_counter_online(unsigned int cpu,
 119                 struct lttng_cpuhp_node *node)
 120 {
 121         struct lttng_perf_counter_field *perf_field =
 122                 container_of(node, struct lttng_perf_counter_field,
 123                                 cpuhp_online);
 124         struct perf_event **events = perf_field->e;
 125         struct perf_event_attr *attr = perf_field->attr;
 126         struct perf_event *pevent;
 127
 128         pevent = wrapper_perf_event_create_kernel_counter(attr,
 129                         cpu, NULL, overflow_callback);
 130         if (!pevent || IS_ERR(pevent))
 131                 return -EINVAL;
 132         if (pevent->state == PERF_EVENT_STATE_ERROR) {
 133                 perf_event_release_kernel(pevent);
 134                 return -EINVAL;
 135         }
 136         barrier();      /* Create perf counter before setting event */
 137         events[cpu] = pevent;
 138         return 0;
 139 }
 140
 141 int lttng_cpuhp_perf_counter_dead(unsigned int cpu,
 142                 struct lttng_cpuhp_node *node)
 143 {
 144         struct lttng_perf_counter_field *perf_field =
 145                 container_of(node, struct lttng_perf_counter_field,
 146                                 cpuhp_prepare);
 147         struct perf_event **events = perf_field->e;
 148         struct perf_event *pevent;
 149
 150         pevent = events[cpu];
 151         events[cpu] = NULL;
 152         barrier();      /* NULLify event before perf counter teardown */
 153         perf_event_release_kernel(pevent);
 154         return 0;
 155 }
 156
 157 #else /* #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,10,0)) */
 158
 159 #ifdef CONFIG_HOTPLUG_CPU
 160
 161 /**
 162  *      lttng_perf_counter_hp_callback - CPU hotplug callback
 163  *      @nb: notifier block
 164  *      @action: hotplug action to take
 165  *      @hcpu: CPU number
 166  *
 167  *      Returns the success/failure of the operation. (%NOTIFY_OK, %NOTIFY_BAD)
 168  *
 169  * We can setup perf counters when the cpu is online (up prepare seems to be too
 170  * soon).
 171  */
 172 static
 173 int lttng_perf_counter_cpu_hp_callback(struct notifier_block *nb,
 174                                                  unsigned long action,
 175                                                  void *hcpu)
 176 {
 177         unsigned int cpu = (unsigned long) hcpu;
 178         struct lttng_perf_counter_field *perf_field =
 179                 container_of(nb, struct lttng_perf_counter_field, nb);
 180         struct perf_event **events = perf_field->e;
 181         struct perf_event_attr *attr = perf_field->attr;
 182         struct perf_event *pevent;
 183
 184         if (!perf_field->hp_enable)
 185                 return NOTIFY_OK;
 186
 187         switch (action) {
 188         case CPU_ONLINE:
 189         case CPU_ONLINE_FROZEN:
 190                 pevent = wrapper_perf_event_create_kernel_counter(attr,
 191                                 cpu, NULL, overflow_callback);
 192                 if (!pevent || IS_ERR(pevent))
 193                         return NOTIFY_BAD;
 194                 if (pevent->state == PERF_EVENT_STATE_ERROR) {
 195                         perf_event_release_kernel(pevent);
 196                         return NOTIFY_BAD;
 197                 }
 198                 barrier();      /* Create perf counter before setting event */
 199                 events[cpu] = pevent;
 200                 break;
 201         case CPU_UP_CANCELED:
 202         case CPU_UP_CANCELED_FROZEN:
 203         case CPU_DEAD:
 204         case CPU_DEAD_FROZEN:
 205                 pevent = events[cpu];
 206                 events[cpu] = NULL;
 207                 barrier();      /* NULLify event before perf counter teardown */
 208                 perf_event_release_kernel(pevent);
 209                 break;
 210         }
 211         return NOTIFY_OK;
 212 }
 213
 214 #endif
 215
 216 #endif /* #else #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,10,0)) */
 217
 218 static const struct lttng_kernel_type_common *field_type =
 219         lttng_kernel_static_type_integer_from_type(uint64_t, __BYTE_ORDER, 10);
 220
 221 int lttng_add_perf_counter_to_ctx(uint32_t type,
 222                                   uint64_t config,
 223                                   const char *name,
 224                                   struct lttng_kernel_ctx **ctx)
 225 {
 226         struct lttng_kernel_ctx_field ctx_field = { 0 };
 227         struct lttng_kernel_event_field *event_field;
 228         struct lttng_perf_counter_field *perf_field;
 229         struct perf_event **events;
 230         struct perf_event_attr *attr;
 231         int ret;
 232         char *name_alloc;
 233
 234         if (lttng_kernel_find_context(*ctx, name))
 235                 return -EEXIST;
 236         name_alloc = kstrdup(name, GFP_KERNEL);
 237         if (!name_alloc) {
 238                 ret = -ENOMEM;
 239                 goto name_alloc_error;
 240         }
 241         event_field = kzalloc(sizeof(*event_field), GFP_KERNEL);
 242         if (!event_field) {
 243                 ret = -ENOMEM;
 244                 goto event_field_alloc_error;
 245         }
 246         event_field->name = name_alloc;
 247         event_field->type = field_type;
 248
 249         events = lttng_kvzalloc(num_possible_cpus() * sizeof(*events), GFP_KERNEL);
 250         if (!events) {
 251                 ret = -ENOMEM;
 252                 goto event_alloc_error;
 253         }
 254
 255         attr = kzalloc(sizeof(struct perf_event_attr), GFP_KERNEL);
 256         if (!attr) {
 257                 ret = -ENOMEM;
 258                 goto error_attr;
 259         }
 260
 261         attr->type = type;
 262         attr->config = config;
 263         attr->size = sizeof(struct perf_event_attr);
 264         attr->pinned = 1;
 265         attr->disabled = 0;
 266
 267         perf_field = kzalloc(sizeof(struct lttng_perf_counter_field), GFP_KERNEL);
 268         if (!perf_field) {
 269                 ret = -ENOMEM;
 270                 goto error_alloc_perf_field;
 271         }
 272         perf_field->e = events;
 273         perf_field->attr = attr;
 274         perf_field->name = name_alloc;
 275         perf_field->event_field = event_field;
 276
 277         ctx_field.event_field = event_field;
 278         ctx_field.get_size = perf_counter_get_size;
 279         ctx_field.record = perf_counter_record;
 280         ctx_field.destroy = lttng_destroy_perf_counter_ctx_field;
 281         ctx_field.priv = perf_field;
 282
 283 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,10,0))
 284
 285         perf_field->cpuhp_prepare.component = LTTNG_CONTEXT_PERF_COUNTERS;
 286         ret = cpuhp_state_add_instance(lttng_hp_prepare,
 287                 &perf_field->cpuhp_prepare.node);
 288         if (ret)
 289                 goto cpuhp_prepare_error;
 290
 291         perf_field->cpuhp_online.component = LTTNG_CONTEXT_PERF_COUNTERS;
 292         ret = cpuhp_state_add_instance(lttng_hp_online,
 293                 &perf_field->cpuhp_online.node);
 294         if (ret)
 295                 goto cpuhp_online_error;
 296
 297 #else   /* #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,10,0)) */
 298         {
 299                 int cpu;
 300
 301 #ifdef CONFIG_HOTPLUG_CPU
 302                 perf_field->nb.notifier_call =
 303                         lttng_perf_counter_cpu_hp_callback;
 304                 perf_field->nb.priority = 0;
 305                 register_cpu_notifier(&perf_field->nb);
 306 #endif
 307                 get_online_cpus();
 308                 for_each_online_cpu(cpu) {
 309                         events[cpu] = wrapper_perf_event_create_kernel_counter(attr,
 310                                                 cpu, NULL, overflow_callback);
 311                         if (!events[cpu] || IS_ERR(events[cpu])) {
 312                                 ret = -EINVAL;
 313                                 goto counter_error;
 314                         }
 315                         if (events[cpu]->state == PERF_EVENT_STATE_ERROR) {
 316                                 ret = -EBUSY;
 317                                 goto counter_busy;
 318                         }
 319                 }
 320                 put_online_cpus();
 321                 perf_field->hp_enable = 1;
 322         }
 323 #endif /* #else #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,10,0)) */
 324
 325         ret = lttng_kernel_context_append(ctx, &ctx_field);
 326         if (ret) {
 327                 ret = -ENOMEM;
 328                 goto append_context_error;
 329         }
 330         return 0;
 331
 332         /* Error handling. */
 333 append_context_error:
 334 #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,10,0))
 335 cpuhp_online_error:
 336         {
 337                 int remove_ret;
 338
 339                 remove_ret = cpuhp_state_remove_instance(lttng_hp_prepare,
 340                                 &perf_field->cpuhp_prepare.node);
 341                 WARN_ON(remove_ret);
 342         }
 343 cpuhp_prepare_error:
 344 #else   /* #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,10,0)) */
 345 counter_busy:
 346 counter_error:
 347         {
 348                 int cpu;
 349
 350                 for_each_online_cpu(cpu) {
 351                         if (events[cpu] && !IS_ERR(events[cpu]))
 352                                 perf_event_release_kernel(events[cpu]);
 353                 }
 354                 put_online_cpus();
 355 #ifdef CONFIG_HOTPLUG_CPU
 356                 unregister_cpu_notifier(&perf_field->nb);
 357 #endif
 358         }
 359 #endif /* #else #if (LTTNG_LINUX_VERSION_CODE >= LTTNG_KERNEL_VERSION(4,10,0)) */
 360         kfree(perf_field);
 361 error_alloc_perf_field:
 362         kfree(attr);
 363 error_attr:
 364         lttng_kvfree(events);
 365 event_alloc_error:
 366         kfree(event_field);
 367 event_field_alloc_error:
 368         kfree(name_alloc);
 369 name_alloc_error:
 370         return ret;
 371 }