lttng-sessiond/ust-app.c

   1 /*
   2  * Copyright (C) 2011 - David Goulet <david.goulet@polymtl.ca>
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public License
   6  * as published by the Free Software Foundation; only version 2
   7  * of the License.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  17  */
  18
  19 #define _GNU_SOURCE
  20 #include <errno.h>
  21 #include <pthread.h>
  22 #include <stdio.h>
  23 #include <stdlib.h>
  24 #include <string.h>
  25 #include <sys/stat.h>
  26 #include <sys/types.h>
  27 #include <unistd.h>
  28
  29 #include <lttngerr.h>
  30 #include <lttng-share.h>
  31
  32 #include "hashtable.h"
  33 #include "ust-app.h"
  34 #include "ust-consumer.h"
  35 #include "ust-ctl.h"
  36
  37 /*
  38  * Delete ust app event safely. RCU read lock must be held before calling
  39  * this function.
  40  */
  41 static void delete_ust_app_event(int sock, struct ust_app_event *ua_event)
  42 {
  43         /* TODO : remove context */
  44         //struct ust_app_ctx *ltctx;
  45         //cds_lfht_for_each_entry(lte->ctx, &iter, ltctx, node) {
  46         //      delete_ust_app_ctx(sock, ltctx);
  47         //}
  48
  49         ustctl_release_object(sock, ua_event->obj);
  50         free(ua_event->obj);
  51         free(ua_event);
  52 }
  53
  54 /*
  55  * Delete ust app stream safely. RCU read lock must be held before calling
  56  * this function.
  57  */
  58 static void delete_ust_app_stream(int sock, struct ltt_ust_stream *stream)
  59 {
  60         ustctl_release_object(sock, stream->obj);
  61         free(stream->obj);
  62         free(stream);
  63 }
  64
  65 /*
  66  * Delete ust app channel safely. RCU read lock must be held before calling
  67  * this function.
  68  */
  69 static void delete_ust_app_channel(int sock, struct ust_app_channel *ua_chan)
  70 {
  71         int ret;
  72         struct cds_lfht_iter iter;
  73         struct ust_app_event *ua_event;
  74         struct ltt_ust_stream *stream, *stmp;
  75
  76         cds_list_for_each_entry_safe(stream, stmp, &ua_chan->streams.head, list) {
  77                 cds_list_del(&stream->list);
  78                 delete_ust_app_stream(sock, stream);
  79         }
  80
  81         /* TODO : remove channel context */
  82         //cds_lfht_for_each_entry(ltc->ctx, &iter, ltctx, node) {
  83         //      hashtable_del(ltc->ctx, &iter);
  84         //      delete_ust_app_ctx(sock, ltctx);
  85         //}
  86         //ret = hashtable_destroy(ltc->ctx);
  87
  88         cds_lfht_for_each_entry(ua_chan->events, &iter, ua_event, node) {
  89                 hashtable_del(ua_chan->events, &iter);
  90                 delete_ust_app_event(sock, ua_event);
  91         }
  92
  93         ret = hashtable_destroy(ua_chan->events);
  94         if (ret < 0) {
  95                 ERR("UST app destroy session hashtable failed");
  96                 goto error;
  97         }
  98         ustctl_release_object(sock, ua_chan->obj);
  99         free(ua_chan->obj);
 100         free(ua_chan);
 101
 102 error:
 103         return;
 104 }
 105
 106 /*
 107  * Delete ust app session safely. RCU read lock must be held before calling
 108  * this function.
 109  */
 110 static void delete_ust_app_session(int sock,
 111                 struct ust_app_session *ua_sess)
 112 {
 113         int ret;
 114         struct cds_lfht_iter iter;
 115         struct ust_app_channel *ua_chan;
 116
 117         if (ua_sess->metadata) {
 118                 ustctl_release_object(sock, ua_sess->metadata->stream_obj);
 119                 free(ua_sess->metadata->stream_obj);
 120                 ustctl_release_object(sock, ua_sess->metadata->obj);
 121                 free(ua_sess->metadata->obj);
 122         }
 123
 124         cds_lfht_for_each_entry(ua_sess->channels, &iter, ua_chan, node) {
 125                 hashtable_del(ua_sess->channels, &iter);
 126                 delete_ust_app_channel(sock, ua_chan);
 127         }
 128
 129         ret = hashtable_destroy(ua_sess->channels);
 130         if (ret < 0) {
 131                 ERR("UST app destroy session hashtable failed");
 132                 goto error;
 133         }
 134
 135 error:
 136         return;
 137 }
 138
 139 /*
 140  * Delete a traceable application structure from the global list. Never call
 141  * this function outside of a call_rcu call.
 142  */
 143 static void delete_ust_app(struct ust_app *app)
 144 {
 145         int ret;
 146         struct cds_lfht_node *node;
 147         struct cds_lfht_iter iter;
 148         struct ust_app_session *ua_sess;
 149         int sock;
 150
 151         rcu_read_lock();
 152
 153         /* Remove from key hash table */
 154         node = hashtable_lookup(ust_app_sock_key_map,
 155                         (void *) ((unsigned long) app->key.sock), sizeof(void *), &iter);
 156         if (node == NULL) {
 157                 /* Not suppose to happen */
 158                 ERR("UST app key %d not found in key hash table", app->key.sock);
 159                 goto end;
 160         }
 161
 162         ret = hashtable_del(ust_app_sock_key_map, &iter);
 163         if (ret) {
 164                 ERR("UST app unable to delete app sock %d from key hash table",
 165                                 app->key.sock);
 166         } else {
 167                 DBG2("UST app pair sock %d key %d deleted",
 168                                 app->key.sock, app->key.pid);
 169         }
 170
 171         /* Socket is already closed at this point */
 172
 173         /* Delete ust app sessions info */
 174         sock = app->key.sock;
 175         app->key.sock = -1;
 176
 177         cds_lfht_for_each_entry(app->sessions, &iter, ua_sess, node) {
 178                 hashtable_del(app->sessions, &iter);
 179                 delete_ust_app_session(app->key.sock, ua_sess);
 180         }
 181
 182         ret = hashtable_destroy(app->sessions);
 183         if (ret < 0) {
 184                 ERR("UST app destroy session hashtable failed");
 185                 goto end;
 186         }
 187
 188         /*
 189          * Wait until we have removed the key from the sock hash table
 190          * before closing this socket, otherwise an application could
 191          * re-use the socket ID and race with the teardown, using the
 192          * same hash table entry.
 193          */
 194         close(sock);
 195
 196         DBG2("UST app pid %d deleted", app->key.pid);
 197         free(app);
 198 end:
 199         rcu_read_unlock();
 200 }
 201
 202 /*
 203  * URCU intermediate call to delete an UST app.
 204  */
 205 static void delete_ust_app_rcu(struct rcu_head *head)
 206 {
 207         struct cds_lfht_node *node =
 208                 caa_container_of(head, struct cds_lfht_node, head);
 209         struct ust_app *app =
 210                 caa_container_of(node, struct ust_app, node);
 211
 212         delete_ust_app(app);
 213 }
 214
 215 /*
 216  * Find an ust_app using the sock and return it. RCU read side lock must be
 217  * held before calling this helper function.
 218  */
 219 static struct ust_app *find_app_by_sock(int sock)
 220 {
 221         struct cds_lfht_node *node;
 222         struct ust_app_key *key;
 223         struct cds_lfht_iter iter;
 224
 225         node = hashtable_lookup(ust_app_sock_key_map,
 226                         (void *)((unsigned long) sock), sizeof(void *), &iter);
 227         if (node == NULL) {
 228                 DBG2("UST app find by sock %d key not found", sock);
 229                 goto error;
 230         }
 231
 232         key = caa_container_of(node, struct ust_app_key, node);
 233
 234         node = hashtable_lookup(ust_app_ht,
 235                         (void *)((unsigned long) key->pid), sizeof(void *), &iter);
 236         if (node == NULL) {
 237                 DBG2("UST app find by sock %d not found", sock);
 238                 goto error;
 239         }
 240         return caa_container_of(node, struct ust_app, node);
 241
 242 error:
 243         return NULL;
 244 }
 245
 246 /*
 247  * Open metadata onto the UST tracer for a UST session.
 248  */
 249 static int open_ust_metadata(struct ust_app *app,
 250                 struct ust_app_session *ua_sess)
 251 {
 252         int ret;
 253         struct lttng_ust_channel_attr uattr;
 254
 255         uattr.overwrite = ua_sess->metadata->attr.overwrite;
 256         uattr.subbuf_size = ua_sess->metadata->attr.subbuf_size;
 257         uattr.num_subbuf = ua_sess->metadata->attr.num_subbuf;
 258         uattr.switch_timer_interval =
 259                 ua_sess->metadata->attr.switch_timer_interval;
 260         uattr.read_timer_interval =
 261                 ua_sess->metadata->attr.read_timer_interval;
 262         uattr.output = ua_sess->metadata->attr.output;
 263
 264         /* UST tracer metadata creation */
 265         ret = ustctl_open_metadata(app->key.sock, ua_sess->handle, &uattr,
 266                         &ua_sess->metadata->obj);
 267         if (ret < 0) {
 268                 ERR("UST app open metadata failed for app pid:%d",
 269                                 app->key.pid);
 270                 goto error;
 271         }
 272
 273 error:
 274         return ret;
 275 }
 276
 277 /*
 278  * Create stream onto the UST tracer for a UST session.
 279  */
 280 static int create_ust_stream(struct ust_app *app,
 281                 struct ust_app_session *ua_sess)
 282 {
 283         int ret;
 284
 285         ret = ustctl_create_stream(app->key.sock, ua_sess->metadata->obj,
 286                         &ua_sess->metadata->stream_obj);
 287         if (ret < 0) {
 288                 ERR("UST create metadata stream failed");
 289                 goto error;
 290         }
 291
 292 error:
 293         return ret;
 294 }
 295
 296 /*
 297  * Create the specified channel onto the UST tracer for a UST session.
 298  */
 299 static int create_ust_channel(struct ust_app *app,
 300                 struct ust_app_session *ua_sess, struct ust_app_channel *ua_chan)
 301 {
 302         int ret;
 303
 304         /* TODO: remove cast and use lttng-ust-abi.h */
 305         ret = ustctl_create_channel(app->key.sock, ua_sess->handle,
 306                         (struct lttng_ust_channel_attr *)&ua_chan->attr, &ua_chan->obj);
 307         if (ret < 0) {
 308                 DBG("Error creating channel %s for app (pid: %d, sock: %d) "
 309                                 "and session handle %d with ret %d",
 310                                 ua_chan->name, app->key.pid, app->key.sock,
 311                                 ua_sess->handle, ret);
 312                 goto error;
 313         }
 314
 315         ua_chan->handle = ua_chan->obj->handle;
 316         ua_chan->attr.shm_fd = ua_chan->obj->shm_fd;
 317         ua_chan->attr.wait_fd = ua_chan->obj->wait_fd;
 318         ua_chan->attr.memory_map_size = ua_chan->obj->memory_map_size;
 319
 320         DBG2("UST app channel %s created successfully for pid:%d and sock:%d",
 321                         ua_chan->name, app->key.pid, app->key.sock);
 322
 323 error:
 324         return ret;
 325 }
 326
 327 /*
 328  * Create the specified event onto the UST tracer for a UST session.
 329  */
 330 static int create_ust_event(struct ust_app *app,
 331                 struct ust_app_session *ua_sess, struct ust_app_channel *ua_chan,
 332                 struct ust_app_event *ua_event)
 333 {
 334         int ret = 0;
 335
 336         /* Create UST event on tracer */
 337         ret = ustctl_create_event(app->key.sock, &ua_event->attr, ua_chan->obj,
 338                         &ua_event->obj);
 339         if (ret < 0) {
 340                 ERR("Error ustctl create event %s for app pid: %d with ret %d",
 341                                 ua_event->attr.name, app->key.pid, ret);
 342                 goto error;
 343         }
 344
 345         ua_event->handle = ua_event->obj->handle;
 346         ua_event->enabled = 1;
 347
 348         DBG2("UST app event %s created successfully for pid:%d",
 349                         ua_event->attr.name, app->key.pid);
 350
 351 error:
 352         return ret;
 353 }
 354
 355 /*
 356  * Alloc new UST app session.
 357  */
 358 static struct ust_app_session *alloc_ust_app_session(void)
 359 {
 360         struct ust_app_session *ua_sess;
 361
 362         /* Init most of the default value by allocating and zeroing */
 363         ua_sess = zmalloc(sizeof(struct ust_app_session));
 364         if (ua_sess == NULL) {
 365                 PERROR("malloc");
 366                 goto error;
 367         }
 368
 369         ua_sess->handle = -1;
 370         ua_sess->channels = hashtable_new_str(0);
 371
 372         return ua_sess;
 373
 374 error:
 375         return NULL;
 376 }
 377
 378 /*
 379  * Alloc new UST app channel.
 380  */
 381 static struct ust_app_channel *alloc_ust_app_channel(char *name,
 382                 struct lttng_ust_channel *attr)
 383 {
 384         struct ust_app_channel *ua_chan;
 385
 386         /* Init most of the default value by allocating and zeroing */
 387         ua_chan = zmalloc(sizeof(struct ust_app_channel));
 388         if (ua_chan == NULL) {
 389                 PERROR("malloc");
 390                 goto error;
 391         }
 392
 393         /* Setup channel name */
 394         strncpy(ua_chan->name, name, sizeof(ua_chan->name));
 395         ua_chan->name[sizeof(ua_chan->name) - 1] = '\0';
 396
 397         ua_chan->handle = -1;
 398         ua_chan->ctx = hashtable_new(0);
 399         ua_chan->events = hashtable_new_str(0);
 400         hashtable_node_init(&ua_chan->node, (void *) ua_chan->name,
 401                         strlen(ua_chan->name));
 402
 403         CDS_INIT_LIST_HEAD(&ua_chan->streams.head);
 404
 405         /* Copy attributes */
 406         if (attr) {
 407                 memcpy(&ua_chan->attr, attr, sizeof(ua_chan->attr));
 408         }
 409
 410         DBG3("UST app channel %s allocated", ua_chan->name);
 411
 412         return ua_chan;
 413
 414 error:
 415         return NULL;
 416 }
 417
 418 /*
 419  * Alloc new UST app event.
 420  */
 421 static struct ust_app_event *alloc_ust_app_event(char *name,
 422                 struct lttng_ust_event *attr)
 423 {
 424         struct ust_app_event *ua_event;
 425
 426         /* Init most of the default value by allocating and zeroing */
 427         ua_event = zmalloc(sizeof(struct ust_app_event));
 428         if (ua_event == NULL) {
 429                 PERROR("malloc");
 430                 goto error;
 431         }
 432
 433         strncpy(ua_event->name, name, sizeof(ua_event->name));
 434         ua_event->name[sizeof(ua_event->name) - 1] = '\0';
 435         ua_event->ctx = hashtable_new(0);
 436         hashtable_node_init(&ua_event->node, (void *) ua_event->name,
 437                         strlen(ua_event->name));
 438
 439         /* Copy attributes */
 440         if (attr) {
 441                 memcpy(&ua_event->attr, attr, sizeof(ua_event->attr));
 442         }
 443
 444         DBG3("UST app event %s allocated", ua_event->name);
 445
 446         return ua_event;
 447
 448 error:
 449         return NULL;
 450 }
 451
 452 /*
 453  * Copy data between an UST app event and a LTT event.
 454  */
 455 static void shadow_copy_event(struct ust_app_event *ua_event,
 456                 struct ltt_ust_event *uevent)
 457 {
 458         strncpy(ua_event->name, uevent->attr.name, sizeof(ua_event->name));
 459         ua_event->name[sizeof(ua_event->name) - 1] = '\0';
 460
 461         /* Copy event attributes */
 462         memcpy(&ua_event->attr, &uevent->attr, sizeof(ua_event->attr));
 463
 464         /* TODO: support copy context */
 465 }
 466
 467 /*
 468  * Copy data between an UST app channel and a LTT channel.
 469  */
 470 static void shadow_copy_channel(struct ust_app_channel *ua_chan,
 471                 struct ltt_ust_channel *uchan)
 472 {
 473         struct cds_lfht_iter iter;
 474         struct cds_lfht_node *ua_event_node;
 475         struct ltt_ust_event *uevent;
 476         struct ust_app_event *ua_event;
 477
 478         DBG2("Shadow copy of UST app channel %s", ua_chan->name);
 479
 480         strncpy(ua_chan->name, uchan->name, sizeof(ua_chan->name));
 481         ua_chan->name[sizeof(ua_chan->name) - 1] = '\0';
 482         /* Copy event attributes */
 483         memcpy(&ua_chan->attr, &uchan->attr, sizeof(ua_chan->attr));
 484
 485         /* TODO: support copy context */
 486
 487         /* Copy all events from ltt ust channel to ust app channel */
 488         cds_lfht_for_each_entry(uchan->events, &iter, uevent, node) {
 489                 struct cds_lfht_iter uiter;
 490
 491                 ua_event_node = hashtable_lookup(ua_chan->events,
 492                                 (void *) uevent->attr.name, strlen(uevent->attr.name),
 493                                 &uiter);
 494                 if (ua_event_node == NULL) {
 495                         DBG2("UST event %s not found on shadow copy channel",
 496                                         uevent->attr.name);
 497                         ua_event = alloc_ust_app_event(uevent->attr.name, &uevent->attr);
 498                         if (ua_event == NULL) {
 499                                 continue;
 500                         }
 501                         shadow_copy_event(ua_event, uevent);
 502                         hashtable_add_unique(ua_chan->events, &ua_event->node);
 503                 }
 504         }
 505
 506         DBG3("Shadow copy channel done");
 507 }
 508
 509 /*
 510  * Copy data between a UST app session and a regular LTT session.
 511  */
 512 static void shadow_copy_session(struct ust_app_session *ua_sess,
 513                 struct ltt_ust_session *usess,
 514                 struct ust_app *app)
 515 {
 516         struct cds_lfht_node *ua_chan_node;
 517         struct cds_lfht_iter iter;
 518         struct ltt_ust_channel *uchan;
 519         struct ust_app_channel *ua_chan;
 520         time_t rawtime;
 521         struct tm *timeinfo;
 522         char datetime[16];
 523         int ret;
 524
 525         /* Get date and time for unique app path */
 526         time(&rawtime);
 527         timeinfo = localtime(&rawtime);
 528         strftime(datetime, sizeof(datetime), "%Y%m%d-%H%M%S", timeinfo);
 529
 530         DBG2("Shadow copy of session handle %d", ua_sess->handle);
 531
 532         ua_sess->uid = usess->uid;
 533
 534         ret = snprintf(ua_sess->path, PATH_MAX,
 535                         "%s/%s-%d-%s",
 536                         usess->pathname, app->name, app->key.pid,
 537                         datetime);
 538         if (ret < 0) {
 539                 PERROR("asprintf UST shadow copy session");
 540                 /* TODO: We cannot return an error from here.. */
 541                 assert(0);
 542         }
 543
 544         /* TODO: support all UST domain */
 545
 546         /* Iterate over all channels in global domain. */
 547         cds_lfht_for_each_entry(usess->domain_global.channels, &iter,
 548                         uchan, node) {
 549                 struct cds_lfht_iter uiter;
 550
 551                 ua_chan_node = hashtable_lookup(ua_sess->channels,
 552                                 (void *)uchan->name, strlen(uchan->name),
 553                                 &uiter);
 554                 if (ua_chan_node != NULL) {
 555                         continue;
 556                 }
 557
 558                 DBG2("Channel %s not found on shadow session copy, creating it",
 559                                 uchan->name);
 560                 ua_chan = alloc_ust_app_channel(uchan->name, &uchan->attr);
 561                 if (ua_chan == NULL) {
 562                         /* malloc failed... continuing */
 563                         continue;
 564                 }
 565
 566                 shadow_copy_channel(ua_chan, uchan);
 567                 hashtable_add_unique(ua_sess->channels, &ua_chan->node);
 568         }
 569 }
 570
 571 static
 572 void __lookup_session_by_app(struct ltt_ust_session *usess,
 573                         struct ust_app *app, struct cds_lfht_iter *iter)
 574 {
 575         /* Get right UST app session from app */
 576         (void) hashtable_lookup(app->sessions,
 577                         (void *) ((unsigned long) usess->uid), sizeof(void *),
 578                         iter);
 579 }
 580
 581 /*
 582  * Return ust app session from the app session hashtable using the UST session
 583  * uid.
 584  */
 585 static struct ust_app_session *lookup_session_by_app(
 586                 struct ltt_ust_session *usess, struct ust_app *app)
 587 {
 588         struct cds_lfht_iter iter;
 589         struct cds_lfht_node *node;
 590
 591         __lookup_session_by_app(usess, app, &iter);
 592         node = hashtable_iter_get_node(&iter);
 593         if (node == NULL) {
 594                 goto error;
 595         }
 596
 597         return caa_container_of(node, struct ust_app_session, node);
 598
 599 error:
 600         return NULL;
 601 }
 602
 603 /*
 604  * Create a UST session onto the tracer of app and add it the session
 605  * hashtable.
 606  *
 607  * Return ust app session or NULL on error.
 608  */
 609 static struct ust_app_session *create_ust_app_session(
 610                 struct ltt_ust_session *usess, struct ust_app *app)
 611 {
 612         int ret;
 613         struct ust_app_session *ua_sess;
 614
 615         ua_sess = lookup_session_by_app(usess, app);
 616         if (ua_sess == NULL) {
 617                 DBG2("UST app pid: %d session uid %d not found, creating it",
 618                                 app->key.pid, usess->uid);
 619                 ua_sess = alloc_ust_app_session();
 620                 if (ua_sess == NULL) {
 621                         /* Only malloc can failed so something is really wrong */
 622                         goto error;
 623                 }
 624                 shadow_copy_session(ua_sess, usess, app);
 625         }
 626
 627         if (ua_sess->handle == -1) {
 628                 ret = ustctl_create_session(app->key.sock);
 629                 if (ret < 0) {
 630                         ERR("Error creating session for app pid %d, sock %d",
 631                                         app->key.pid, app->key.sock);
 632                         /* TODO: free() ua_sess */
 633                         goto error;
 634                 }
 635
 636                 DBG2("UST app ustctl create session handle %d", ret);
 637                 ua_sess->handle = ret;
 638
 639                 /* Add ust app session to app's HT */
 640                 hashtable_node_init(&ua_sess->node,
 641                                 (void *)((unsigned long) ua_sess->uid), sizeof(void *));
 642                 hashtable_add_unique(app->sessions, &ua_sess->node);
 643
 644                 DBG2("UST app session created successfully with handle %d", ret);
 645         }
 646
 647         return ua_sess;
 648
 649 error:
 650         return NULL;
 651 }
 652
 653 /*
 654  * Create UST app channel and create it on the tracer.
 655  */
 656 static struct ust_app_channel *create_ust_app_channel(
 657                 struct ust_app_session *ua_sess, struct ltt_ust_channel *uchan,
 658                 struct ust_app *app)
 659 {
 660         int ret = 0;
 661         struct cds_lfht_iter iter;
 662         struct cds_lfht_node *ua_chan_node;
 663         struct ust_app_channel *ua_chan;
 664
 665         /* Lookup channel in the ust app session */
 666         ua_chan_node = hashtable_lookup(ua_sess->channels,
 667                         (void *)uchan->name, strlen(uchan->name), &iter);
 668         if (ua_chan_node == NULL) {
 669                 DBG2("Unable to find channel %s in ust session uid %u",
 670                                 uchan->name, ua_sess->uid);
 671                 ua_chan = alloc_ust_app_channel(uchan->name, &uchan->attr);
 672                 if (ua_chan == NULL) {
 673                         goto error;
 674                 }
 675                 shadow_copy_channel(ua_chan, uchan);
 676
 677                 hashtable_add_unique(ua_sess->channels, &ua_chan->node);
 678         } else {
 679                 ua_chan = caa_container_of(ua_chan_node, struct ust_app_channel, node);
 680         }
 681
 682         ret = create_ust_channel(app, ua_sess, ua_chan);
 683         if (ret < 0) {
 684                 goto error;
 685         }
 686
 687         return ua_chan;
 688
 689 error:
 690         return NULL;
 691 }
 692
 693 /*
 694  * Create UST app event and create it on the tracer side.
 695  */
 696 static struct ust_app_event *create_ust_app_event(
 697                 struct ust_app_session *ua_sess, struct ust_app_channel *ua_chan,
 698                 struct ltt_ust_event *uevent, struct ust_app *app)
 699 {
 700         int ret;
 701         struct cds_lfht_iter iter;
 702         struct cds_lfht_node *ua_event_node;
 703         struct ust_app_event *ua_event;
 704
 705         /* Get event node */
 706         ua_event_node = hashtable_lookup(ua_chan->events,
 707                         (void *)uevent->attr.name, strlen(uevent->attr.name), &iter);
 708         if (ua_event_node == NULL) {
 709                 DBG2("UST app event %s not found, creating it", uevent->attr.name);
 710                 /* Does not exist so create one */
 711                 ua_event = alloc_ust_app_event(uevent->attr.name, &uevent->attr);
 712                 if (ua_event == NULL) {
 713                         /* Only malloc can failed so something is really wrong */
 714                         goto error;
 715                 }
 716                 shadow_copy_event(ua_event, uevent);
 717
 718                 hashtable_add_unique(ua_chan->events, &ua_event->node);
 719         } else {
 720                 ua_event = caa_container_of(ua_event_node, struct ust_app_event, node);
 721         }
 722
 723         ret = create_ust_event(app, ua_sess, ua_chan, ua_event);
 724         if (ret < 0) {
 725                 goto error;
 726         }
 727
 728         return ua_event;
 729
 730 error:
 731         return NULL;
 732 }
 733
 734 /*
 735  * Create UST metadata and open it on the tracer side.
 736  */
 737 static int create_ust_app_metadata(struct ust_app_session *ua_sess,
 738                 char *pathname, struct ust_app *app)
 739 {
 740         int ret = 0;
 741
 742         if (ua_sess->metadata == NULL) {
 743                 /* Allocate UST metadata */
 744                 ua_sess->metadata = trace_ust_create_metadata(pathname);
 745                 if (ua_sess->metadata == NULL) {
 746                         ERR("UST app session %d creating metadata failed",
 747                                         ua_sess->handle);
 748                         goto error;
 749                 }
 750
 751                 ret = open_ust_metadata(app, ua_sess);
 752                 if (ret < 0) {
 753                         goto error;
 754                 }
 755
 756                 DBG2("UST metadata opened for app pid %d", app->key.pid);
 757         }
 758
 759         /* Open UST metadata stream */
 760         if (ua_sess->metadata->stream_obj == NULL) {
 761                 ret = create_ust_stream(app, ua_sess);
 762                 if (ret < 0) {
 763                         goto error;
 764                 }
 765
 766                 ret = mkdir(ua_sess->path, S_IRWXU | S_IRWXG);
 767                 if (ret < 0) {
 768                         PERROR("mkdir UST metadata");
 769                         goto error;
 770                 }
 771
 772                 ret = snprintf(ua_sess->metadata->pathname, PATH_MAX,
 773                                 "%s/metadata", ua_sess->path);
 774                 if (ret < 0) {
 775                         PERROR("asprintf UST create stream");
 776                         goto error;
 777                 }
 778
 779                 DBG2("UST metadata stream object created for app pid %d",
 780                                 app->key.pid);
 781         } else {
 782                 ERR("Attempting to create stream without metadata opened");
 783                 goto error;
 784         }
 785
 786         return 0;
 787
 788 error:
 789         return -1;
 790 }
 791
 792 /*
 793  * Return pointer to traceable apps list.
 794  */
 795 struct cds_lfht *ust_app_get_ht(void)
 796 {
 797         return ust_app_ht;
 798 }
 799
 800 /*
 801  * Return ust app pointer or NULL if not found.
 802  */
 803 struct ust_app *ust_app_find_by_pid(pid_t pid)
 804 {
 805         struct cds_lfht_node *node;
 806         struct cds_lfht_iter iter;
 807
 808         rcu_read_lock();
 809         node = hashtable_lookup(ust_app_ht,
 810                         (void *)((unsigned long) pid), sizeof(void *), &iter);
 811         if (node == NULL) {
 812                 DBG2("UST app no found with pid %d", pid);
 813                 goto error;
 814         }
 815         rcu_read_unlock();
 816
 817         DBG2("Found UST app by pid %d", pid);
 818
 819         return caa_container_of(node, struct ust_app, node);
 820
 821 error:
 822         rcu_read_unlock();
 823         return NULL;
 824 }
 825
 826 /*
 827  * Using pid and uid (of the app), allocate a new ust_app struct and
 828  * add it to the global traceable app list.
 829  *
 830  * On success, return 0, else return malloc ENOMEM.
 831  */
 832 int ust_app_register(struct ust_register_msg *msg, int sock)
 833 {
 834         struct ust_app *lta;
 835
 836         lta = zmalloc(sizeof(struct ust_app));
 837         if (lta == NULL) {
 838                 PERROR("malloc");
 839                 return -ENOMEM;
 840         }
 841
 842         lta->ppid = msg->ppid;
 843         lta->uid = msg->uid;
 844         lta->gid = msg->gid;
 845         lta->v_major = msg->major;
 846         lta->v_minor = msg->minor;
 847         strncpy(lta->name, msg->name, sizeof(lta->name));
 848         lta->name[16] = '\0';
 849         lta->sessions = hashtable_new(0);
 850
 851         /* Set key map */
 852         lta->key.pid = msg->pid;
 853         hashtable_node_init(&lta->node, (void *)((unsigned long)lta->key.pid),
 854                         sizeof(void *));
 855         lta->key.sock = sock;
 856         hashtable_node_init(&lta->key.node, (void *)((unsigned long)lta->key.sock),
 857                         sizeof(void *));
 858
 859         rcu_read_lock();
 860         hashtable_add_unique(ust_app_sock_key_map, &lta->key.node);
 861         hashtable_add_unique(ust_app_ht, &lta->node);
 862         rcu_read_unlock();
 863
 864         DBG("App registered with pid:%d ppid:%d uid:%d gid:%d sock:%d name:%s"
 865                         " (version %d.%d)", lta->key.pid, lta->ppid, lta->uid, lta->gid,
 866                         lta->key.sock, lta->name, lta->v_major, lta->v_minor);
 867
 868         return 0;
 869 }
 870
 871 /*
 872  * Unregister app by removing it from the global traceable app list and freeing
 873  * the data struct.
 874  *
 875  * The socket is already closed at this point so no close to sock.
 876  */
 877 void ust_app_unregister(int sock)
 878 {
 879         struct ust_app *lta;
 880         struct cds_lfht_node *node;
 881         struct cds_lfht_iter iter;
 882
 883         rcu_read_lock();
 884         lta = find_app_by_sock(sock);
 885         if (lta == NULL) {
 886                 ERR("Unregister app sock %d not found!", sock);
 887                 goto error;
 888         }
 889
 890         DBG("PID %d unregistering with sock %d", lta->key.pid, sock);
 891
 892         /* Get the node reference for a call_rcu */
 893         node = hashtable_lookup(ust_app_ht,
 894                         (void *)((unsigned long) lta->key.pid), sizeof(void *), &iter);
 895         if (node == NULL) {
 896                 ERR("Unable to find app sock %d by pid %d", sock, lta->key.pid);
 897                 goto error;
 898         }
 899
 900         hashtable_del(ust_app_ht, &iter);
 901         call_rcu(&node->head, delete_ust_app_rcu);
 902 error:
 903         rcu_read_unlock();
 904         return;
 905 }
 906
 907 /*
 908  * Return traceable_app_count
 909  */
 910 unsigned long ust_app_list_count(void)
 911 {
 912         unsigned long count;
 913
 914         rcu_read_lock();
 915         count = hashtable_get_count(ust_app_ht);
 916         rcu_read_unlock();
 917
 918         return count;
 919 }
 920
 921 /*
 922  * Fill events array with all events name of all registered apps.
 923  */
 924 int ust_app_list_events(struct lttng_event **events)
 925 {
 926         int ret, handle;
 927         size_t nbmem, count = 0;
 928         struct cds_lfht_iter iter;
 929         struct ust_app *app;
 930         struct lttng_event *tmp;
 931
 932         nbmem = UST_APP_EVENT_LIST_SIZE;
 933         tmp = zmalloc(nbmem * sizeof(struct lttng_event));
 934         if (tmp == NULL) {
 935                 PERROR("zmalloc ust app events");
 936                 ret = -ENOMEM;
 937                 goto error;
 938         }
 939
 940         rcu_read_lock();
 941
 942         cds_lfht_for_each_entry(ust_app_ht, &iter, app, node) {
 943                 handle = ustctl_tracepoint_list(app->key.sock);
 944                 if (handle < 0) {
 945                         ERR("UST app list events getting handle failed for app pid %d",
 946                                         app->key.pid);
 947                         continue;
 948                 }
 949
 950                 while ((ret = ustctl_tracepoint_list_get(app->key.sock, handle,
 951                                                 tmp[count].name)) != -ENOENT) {
 952                         if (count > nbmem) {
 953                                 DBG2("Reallocating event list from %zu to %zu bytes", nbmem,
 954                                                 nbmem + UST_APP_EVENT_LIST_SIZE);
 955                                 nbmem += UST_APP_EVENT_LIST_SIZE;
 956                                 tmp = realloc(tmp, nbmem);
 957                                 if (tmp == NULL) {
 958                                         PERROR("realloc ust app events");
 959                                         ret = -ENOMEM;
 960                                         goto rcu_error;
 961                                 }
 962                         }
 963
 964                         tmp[count].type = LTTNG_UST_TRACEPOINT;
 965                         tmp[count].pid = app->key.pid;
 966                         tmp[count].enabled = -1;
 967                         count++;
 968                 }
 969         }
 970
 971         ret = count;
 972         *events = tmp;
 973
 974         DBG2("UST app list events done (%zu events)", count);
 975
 976 rcu_error:
 977         rcu_read_unlock();
 978 error:
 979         return ret;
 980 }
 981
 982 /*
 983  * Free and clean all traceable apps of the global list.
 984  */
 985 void ust_app_clean_list(void)
 986 {
 987         int ret;
 988         struct cds_lfht_node *node;
 989         struct cds_lfht_iter iter;
 990         struct ust_app *app;
 991
 992         DBG2("UST app cleaning registered apps hash table");
 993
 994         rcu_read_lock();
 995
 996         cds_lfht_for_each(ust_app_ht, &iter, node) {
 997                 app = caa_container_of(node, struct ust_app, node);
 998
 999                 ret = hashtable_del(ust_app_ht, &iter);
1000                 if (!ret) {
1001                         call_rcu(&node->head, delete_ust_app_rcu);
1002                 }
1003         }
1004
1005         hashtable_destroy(ust_app_ht);
1006         hashtable_destroy(ust_app_sock_key_map);
1007
1008         rcu_read_unlock();
1009 }
1010
1011 /*
1012  * Init UST app hash table.
1013  */
1014 void ust_app_ht_alloc(void)
1015 {
1016         ust_app_ht = hashtable_new(0);
1017         ust_app_sock_key_map = hashtable_new(0);
1018 }
1019
1020 /*
1021  * For a specific UST session, create the channel for all registered apps.
1022  */
1023 int ust_app_create_channel_all(struct ltt_ust_session *usess,
1024                 struct ltt_ust_channel *uchan)
1025 {
1026         int ret = 0;
1027         struct cds_lfht_iter iter;
1028         struct ust_app *app;
1029         struct ust_app_session *ua_sess;
1030         struct ust_app_channel *ua_chan;
1031
1032         if (usess == NULL || uchan == NULL) {
1033                 ERR("Adding UST global channel to NULL values");
1034                 ret = -1;
1035                 goto error;
1036         }
1037
1038         DBG2("UST app adding channel %s to global domain for session uid %d",
1039                         uchan->name, usess->uid);
1040
1041         rcu_read_lock();
1042
1043         /* For every registered applications */
1044         cds_lfht_for_each_entry(ust_app_ht, &iter, app, node) {
1045                 /* Create session on the tracer side and add it to app session HT */
1046                 ua_sess = create_ust_app_session(usess, app);
1047                 if (ua_sess == NULL) {
1048                         continue;
1049                 }
1050
1051                 /* Create channel onto application */
1052                 ua_chan = create_ust_app_channel(ua_sess, uchan, app);
1053                 if (ua_chan == NULL) {
1054                         continue;
1055                 }
1056         }
1057
1058         rcu_read_unlock();
1059
1060 error:
1061         return ret;
1062 }
1063
1064 /*
1065  * For a specific UST session and UST channel, create the event for all
1066  * registered apps.
1067  */
1068 int ust_app_create_event_all(struct ltt_ust_session *usess,
1069                 struct ltt_ust_channel *uchan, struct ltt_ust_event *uevent)
1070 {
1071         int ret = 0;
1072         struct cds_lfht_iter iter;
1073         struct cds_lfht_node *ua_chan_node;
1074         struct ust_app *app;
1075         struct ust_app_session *ua_sess;
1076         struct ust_app_channel *ua_chan;
1077         struct ust_app_event *ua_event;
1078
1079         DBG("UST app creating event %s for all apps for session uid %d",
1080                         uevent->attr.name, usess->uid);
1081
1082         rcu_read_lock();
1083
1084         /* For all registered applications */
1085         cds_lfht_for_each_entry(ust_app_ht, &iter, app, node) {
1086                 struct cds_lfht_iter uiter;
1087
1088                 /* Create session on the tracer side and add it to app session HT */
1089                 ua_sess = create_ust_app_session(usess, app);
1090                 if (ua_sess == NULL) {
1091                         continue;
1092                 }
1093
1094                 /* Lookup channel in the ust app session */
1095                 ua_chan_node = hashtable_lookup(ua_sess->channels,
1096                                 (void *)uchan->name, strlen(uchan->name),
1097                                 &uiter);
1098                 if (ua_chan_node == NULL) {
1099                         ERR("Channel %s not found in session uid %d. Skipping",
1100                                         uchan->name, usess->uid);
1101                         continue;
1102                 }
1103                 ua_chan = caa_container_of(ua_chan_node, struct ust_app_channel, node);
1104
1105                 ua_event = create_ust_app_event(ua_sess, ua_chan, uevent, app);
1106                 if (ua_event == NULL) {
1107                         continue;
1108                 }
1109         }
1110
1111         rcu_read_unlock();
1112
1113         return ret;
1114 }
1115
1116 /*
1117  * Start tracing for a specific UST session and app.
1118  */
1119 int ust_app_start_trace(struct ltt_ust_session *usess, struct ust_app *app)
1120 {
1121         int ret = 0;
1122         struct cds_lfht_iter iter;
1123         struct ust_app_session *ua_sess;
1124         struct ust_app_channel *ua_chan;
1125         struct ltt_ust_stream *ustream;
1126
1127         DBG("Starting tracing for ust app pid %d", app->key.pid);
1128
1129         rcu_read_lock();
1130
1131         ua_sess = lookup_session_by_app(usess, app);
1132         if (ua_sess == NULL) {
1133                 /* Only malloc can failed so something is really wrong */
1134                 goto error_rcu_unlock;
1135         }
1136
1137         /* upon restart, we skip the setup, already done */
1138         if (ua_sess->started)
1139                 goto skip_setup;
1140
1141         ret = create_ust_app_metadata(ua_sess, usess->pathname, app);
1142         if (ret < 0) {
1143                 goto error_rcu_unlock;
1144         }
1145
1146         /* For each channel */
1147         cds_lfht_for_each_entry(ua_sess->channels, &iter, ua_chan, node) {
1148                 /* Create all streams */
1149                 while (1) {
1150                         /* Create UST stream */
1151                         ustream = zmalloc(sizeof(*ustream));
1152                         if (ustream == NULL) {
1153                                 PERROR("zmalloc ust stream");
1154                                 goto error_rcu_unlock;
1155                         }
1156
1157                         ret = ustctl_create_stream(app->key.sock, ua_chan->obj,
1158                                         &ustream->obj);
1159                         if (ret < 0) {
1160                                 /* Got all streams */
1161                                 break;
1162                         }
1163                         ustream->handle = ustream->obj->handle;
1164
1165                         /* Order is important */
1166                         cds_list_add_tail(&ustream->list, &ua_chan->streams.head);
1167                         ret = snprintf(ustream->pathname, PATH_MAX, "%s/%s_%u",
1168                                         ua_sess->path, ua_chan->name,
1169                                         ua_chan->streams.count++);
1170                         if (ret < 0) {
1171                                 PERROR("asprintf UST create stream");
1172                                 continue;
1173                         }
1174                         DBG2("UST stream %d ready at %s", ua_chan->streams.count,
1175                                         ustream->pathname);
1176                 }
1177         }
1178
1179         /* Setup UST consumer socket and send fds to it */
1180         ret = ust_consumer_send_session(ust_consumer_fd, ua_sess);
1181         if (ret < 0) {
1182                 goto error_rcu_unlock;
1183         }
1184         ua_sess->started = 1;
1185
1186 skip_setup:
1187         /* This start the UST tracing */
1188         ret = ustctl_start_session(app->key.sock, ua_sess->handle);
1189         if (ret < 0) {
1190                 ERR("Error starting tracing for app pid: %d", app->key.pid);
1191                 goto error_rcu_unlock;
1192         }
1193
1194         rcu_read_unlock();
1195
1196         /* Quiescent wait after starting trace */
1197         ustctl_wait_quiescent(app->key.sock);
1198
1199         return 0;
1200
1201 error_rcu_unlock:
1202         rcu_read_unlock();
1203         return -1;
1204 }
1205
1206 /*
1207  * Stop tracing for a specific UST session and app.
1208  */
1209 int ust_app_stop_trace(struct ltt_ust_session *usess, struct ust_app *app)
1210 {
1211         int ret = 0;
1212         struct ust_app_session *ua_sess;
1213
1214         DBG("Stopping tracing for ust app pid %d", app->key.pid);
1215
1216         rcu_read_lock();
1217
1218         ua_sess = lookup_session_by_app(usess, app);
1219         if (ua_sess == NULL) {
1220                 /* Only malloc can failed so something is really wrong */
1221                 goto error_rcu_unlock;
1222         }
1223
1224 #if 0   /* only useful when periodical flush will be supported */
1225         /* need to keep a handle on shm in session for this. */
1226         /* Flush all buffers before stopping */
1227         ret = ustctl_flush_buffer(usess->sock, usess->metadata->obj);
1228         if (ret < 0) {
1229                 ERR("UST metadata flush failed");
1230         }
1231
1232         cds_list_for_each_entry(ustchan, &usess->channels.head, list) {
1233                 ret = ustctl_flush_buffer(usess->sock, ustchan->obj);
1234                 if (ret < 0) {
1235                         ERR("UST flush buffer error");
1236                 }
1237         }
1238 #endif
1239
1240         /* This inhibits UST tracing */
1241         ret = ustctl_stop_session(app->key.sock, ua_sess->handle);
1242         if (ret < 0) {
1243                 ERR("Error stopping tracing for app pid: %d", app->key.pid);
1244                 goto error_rcu_unlock;
1245         }
1246
1247         rcu_read_unlock();
1248
1249         /* Quiescent wait after stopping trace */
1250         ustctl_wait_quiescent(app->key.sock);
1251
1252         return 0;
1253
1254 error_rcu_unlock:
1255         rcu_read_unlock();
1256         return -1;
1257 }
1258
1259 /*
1260  * Destroy a specific UST session in apps.
1261  */
1262 int ust_app_destroy_trace(struct ltt_ust_session *usess, struct ust_app *app)
1263 {
1264         struct ust_app_session *ua_sess;
1265         struct lttng_ust_object_data obj;
1266         struct cds_lfht_iter iter;
1267         struct cds_lfht_node *node;
1268
1269         DBG("Destroy tracing for ust app pid %d", app->key.pid);
1270
1271         rcu_read_lock();
1272
1273         __lookup_session_by_app(usess, app, &iter);
1274         node = hashtable_iter_get_node(&iter);
1275         if (node == NULL) {
1276                 /* Only malloc can failed so something is really wrong */
1277                 goto error_rcu_unlock;
1278         }
1279         ua_sess = caa_container_of(node, struct ust_app_session, node);
1280         hashtable_del(app->sessions, &iter);
1281         delete_ust_app_session(app->key.sock, ua_sess);
1282         obj.handle = ua_sess->handle;
1283         obj.shm_fd = -1;
1284         obj.wait_fd = -1;
1285         obj.memory_map_size = 0;
1286         ustctl_release_object(app->key.sock, &obj);
1287
1288         rcu_read_unlock();
1289
1290         /* Quiescent wait after stopping trace */
1291         ustctl_wait_quiescent(app->key.sock);
1292
1293         return 0;
1294
1295 error_rcu_unlock:
1296         rcu_read_unlock();
1297         return -1;
1298 }
1299
1300 /*
1301  * Start tracing for the UST session.
1302  */
1303 int ust_app_start_trace_all(struct ltt_ust_session *usess)
1304 {
1305         int ret = 0;
1306         struct cds_lfht_iter iter;
1307         struct ust_app *app;
1308
1309         DBG("Starting all UST traces");
1310
1311         rcu_read_lock();
1312
1313         cds_lfht_for_each_entry(ust_app_ht, &iter, app, node) {
1314                 ret = ust_app_start_trace(usess, app);
1315                 if (ret < 0) {
1316                         /* Continue to next apps even on error */
1317                         continue;
1318                 }
1319         }
1320
1321         rcu_read_unlock();
1322
1323         return 0;
1324 }
1325
1326 /*
1327  * Start tracing for the UST session.
1328  */
1329 int ust_app_stop_trace_all(struct ltt_ust_session *usess)
1330 {
1331         int ret = 0;
1332         struct cds_lfht_iter iter;
1333         struct ust_app *app;
1334
1335         DBG("Stopping all UST traces");
1336
1337         rcu_read_lock();
1338
1339         cds_lfht_for_each_entry(ust_app_ht, &iter, app, node) {
1340                 ret = ust_app_stop_trace(usess, app);
1341                 if (ret < 0) {
1342                         /* Continue to next apps even on error */
1343                         continue;
1344                 }
1345         }
1346
1347         rcu_read_unlock();
1348
1349         return 0;
1350 }
1351
1352 /*
1353  * Destroy app UST session.
1354  */
1355 int ust_app_destroy_trace_all(struct ltt_ust_session *usess)
1356 {
1357         int ret = 0;
1358         struct cds_lfht_iter iter;
1359         struct ust_app *app;
1360
1361         DBG("Destroy all UST traces");
1362
1363         rcu_read_lock();
1364
1365         cds_lfht_for_each_entry(ust_app_ht, &iter, app, node) {
1366                 ret = ust_app_destroy_trace(usess, app);
1367                 if (ret < 0) {
1368                         /* Continue to next apps even on error */
1369                         continue;
1370                 }
1371         }
1372
1373         rcu_read_unlock();
1374
1375         return 0;
1376 }
1377
1378 /*
1379  * Add channels/events from UST global domain to registered apps at sock.
1380  */
1381 void ust_app_global_update(struct ltt_ust_session *usess, int sock)
1382 {
1383         int ret = 0;
1384         struct cds_lfht_iter iter;
1385         struct ust_app *app;
1386         struct ust_app_session *ua_sess;
1387         struct ust_app_channel *ua_chan;
1388         struct ust_app_event *ua_event;
1389
1390         if (usess == NULL) {
1391                 ERR("No UST session on global update. Returning");
1392                 goto error;
1393         }
1394
1395         DBG2("UST app global update for app sock %d for session uid %d", sock,
1396                         usess->uid);
1397
1398         rcu_read_lock();
1399
1400         app = find_app_by_sock(sock);
1401         if (app == NULL) {
1402                 ERR("Failed to update app sock %d", sock);
1403                 goto error;
1404         }
1405
1406         ua_sess = create_ust_app_session(usess, app);
1407         if (ua_sess == NULL) {
1408                 goto error;
1409         }
1410
1411         /*
1412          * We can iterate safely here over all UST app session sicne the create ust
1413          * app session above made a shadow copy of the UST global domain from the
1414          * ltt ust session.
1415          */
1416         cds_lfht_for_each_entry(ua_sess->channels, &iter, ua_chan, node) {
1417                 ret = create_ust_channel(app, ua_sess, ua_chan);
1418                 if (ret < 0) {
1419                         /* FIXME: Should we quit here or continue... */
1420                         continue;
1421                 }
1422
1423                 /* For each events */
1424                 cds_lfht_for_each_entry(ua_chan->events, &iter, ua_event, node) {
1425                         ret = create_ust_event(app, ua_sess, ua_chan, ua_event);
1426                         if (ret < 0) {
1427                                 /* FIXME: Should we quit here or continue... */
1428                                 continue;
1429                         }
1430                 }
1431         }
1432
1433         if (usess->start_trace) {
1434                 ret = ust_app_start_trace(usess, app);
1435                 if (ret < 0) {
1436                         goto error;
1437                 }
1438
1439                 DBG2("UST trace started for app pid %d", app->key.pid);
1440         }
1441
1442 error:
1443         rcu_read_unlock();
1444         return;
1445 }