sessiond: agent: enable events matching event notifiers
[lttng-tools.git] / src / bin / lttng-sessiond / agent.c
1 /*
2 * Copyright (C) 2013 David Goulet <dgoulet@efficios.com>
3 * Copyright (C) 2016 Jérémie Galarneau <jeremie.galarneau@efficios.com>
4 *
5 * SPDX-License-Identifier: GPL-2.0-only
6 *
7 */
8
9 #define _LGPL_SOURCE
10 #include <assert.h>
11 #include <urcu/uatomic.h>
12 #include <urcu/rculist.h>
13
14 #include <lttng/event-rule/event-rule.h>
15 #include <lttng/event-rule/event-rule-internal.h>
16 #include <lttng/event-rule/tracepoint.h>
17 #include <lttng/condition/condition.h>
18 #include <lttng/condition/event-rule.h>
19 #include <lttng/domain-internal.h>
20
21 #include <common/common.h>
22 #include <common/sessiond-comm/agent.h>
23
24 #include <common/compat/endian.h>
25
26 #include "agent.h"
27 #include "ust-app.h"
28 #include "utils.h"
29 #include "common/error.h"
30
31 #define AGENT_RET_CODE_INDEX(code) (code - AGENT_RET_CODE_SUCCESS)
32
33 /*
34 * Agent application context representation.
35 */
36 struct agent_app_ctx {
37 char *provider_name;
38 char *ctx_name;
39
40 /* agent_app_ctx are part of the agent app_ctx_list. */
41 struct cds_list_head list_node;
42
43 /* For call_rcu teardown. */
44 struct rcu_head rcu_node;
45 };
46
47 /*
48 * Human readable agent return code.
49 */
50 static const char *error_string_array[] = {
51 [ AGENT_RET_CODE_INDEX(AGENT_RET_CODE_SUCCESS) ] = "Success",
52 [ AGENT_RET_CODE_INDEX(AGENT_RET_CODE_INVALID) ] = "Invalid command",
53 [ AGENT_RET_CODE_INDEX(AGENT_RET_CODE_UNKNOWN_NAME) ] = "Unknown logger name",
54
55 /* Last element */
56 [ AGENT_RET_CODE_INDEX(AGENT_RET_CODE_NR) ] = "Unknown code",
57 };
58
59 static
60 void log_reply_code(uint32_t in_reply_ret_code)
61 {
62 int level = PRINT_DBG3;
63 /*
64 * reply_ret_code and in_reply_ret_code are kept separate to have a
65 * sanitized value (used to retrieve the human readable string) and the
66 * original value which is logged as-is.
67 */
68 uint32_t reply_ret_code = in_reply_ret_code;
69
70 if (reply_ret_code < AGENT_RET_CODE_SUCCESS ||
71 reply_ret_code >= AGENT_RET_CODE_NR) {
72 reply_ret_code = AGENT_RET_CODE_NR;
73 level = PRINT_ERR;
74 }
75
76 LOG(level, "Agent replied with retcode: %s (%"PRIu32")",
77 error_string_array[AGENT_RET_CODE_INDEX(
78 reply_ret_code)],
79 in_reply_ret_code);
80 }
81
82 /*
83 * Match function for the events hash table lookup by name.
84 */
85 static int ht_match_event_by_name(struct cds_lfht_node *node,
86 const void *_key)
87 {
88 struct agent_event *event;
89 const struct agent_ht_key *key;
90
91 assert(node);
92 assert(_key);
93
94 event = caa_container_of(node, struct agent_event, node.node);
95 key = _key;
96
97 /* Match 1 elements of the key: name. */
98
99 /* Event name */
100 if (strncmp(event->name, key->name, sizeof(event->name)) != 0) {
101 goto no_match;
102 }
103 /* Match. */
104 return 1;
105
106 no_match:
107 return 0;
108 }
109
110 /*
111 * Match function for the events hash table lookup by name, log level and
112 * filter expression.
113 */
114 static int ht_match_event(struct cds_lfht_node *node,
115 const void *_key)
116 {
117 struct agent_event *event;
118 const struct agent_ht_key *key;
119 int ll_match;
120
121 assert(node);
122 assert(_key);
123
124 event = caa_container_of(node, struct agent_event, node.node);
125 key = _key;
126
127 /* Match 2 elements of the key: name and loglevel. */
128
129 /* Event name */
130 if (strncmp(event->name, key->name, sizeof(event->name)) != 0) {
131 goto no_match;
132 }
133
134 /* Event loglevel value and type. */
135 ll_match = loglevels_match(event->loglevel_type,
136 event->loglevel_value, key->loglevel_type,
137 key->loglevel_value, LTTNG_EVENT_LOGLEVEL_ALL);
138
139 if (!ll_match) {
140 goto no_match;
141 }
142
143 /* Filter expression */
144 if (!!event->filter_expression != !!key->filter_expression) {
145 /* One has a filter expression, the other does not */
146 goto no_match;
147 }
148
149 if (event->filter_expression) {
150 if (strncmp(event->filter_expression, key->filter_expression,
151 strlen(event->filter_expression)) != 0) {
152 goto no_match;
153 }
154 }
155
156 return 1;
157
158 no_match:
159 return 0;
160 }
161
162 /*
163 * Add unique agent event based on the event name and loglevel.
164 */
165 static void add_unique_agent_event(struct lttng_ht *ht,
166 struct agent_event *event)
167 {
168 struct cds_lfht_node *node_ptr;
169 struct agent_ht_key key;
170
171 assert(ht);
172 assert(ht->ht);
173 assert(event);
174
175 key.name = event->name;
176 key.loglevel_value = event->loglevel_value;
177 key.loglevel_type = event->loglevel_type;
178 key.filter_expression = event->filter_expression;
179
180 node_ptr = cds_lfht_add_unique(ht->ht,
181 ht->hash_fct(event->node.key, lttng_ht_seed),
182 ht_match_event, &key, &event->node.node);
183 assert(node_ptr == &event->node.node);
184 }
185
186 /*
187 * URCU delayed agent event reclaim.
188 */
189 static void destroy_event_agent_rcu(struct rcu_head *head)
190 {
191 struct lttng_ht_node_str *node =
192 caa_container_of(head, struct lttng_ht_node_str, head);
193 struct agent_event *event =
194 caa_container_of(node, struct agent_event, node);
195
196 agent_destroy_event(event);
197 }
198
199 /*
200 * URCU delayed agent app reclaim.
201 */
202 static void destroy_app_agent_rcu(struct rcu_head *head)
203 {
204 struct lttng_ht_node_ulong *node =
205 caa_container_of(head, struct lttng_ht_node_ulong, head);
206 struct agent_app *app =
207 caa_container_of(node, struct agent_app, node);
208
209 free(app);
210 }
211
212 /*
213 * Communication with the agent. Send the message header to the given socket in
214 * big endian.
215 *
216 * Return 0 on success or else a negative errno message of sendmsg() op.
217 */
218 static int send_header(struct lttcomm_sock *sock, uint64_t data_size,
219 uint32_t cmd, uint32_t cmd_version)
220 {
221 int ret;
222 ssize_t size;
223 struct lttcomm_agent_hdr msg;
224
225 assert(sock);
226
227 memset(&msg, 0, sizeof(msg));
228 msg.data_size = htobe64(data_size);
229 msg.cmd = htobe32(cmd);
230 msg.cmd_version = htobe32(cmd_version);
231
232 size = sock->ops->sendmsg(sock, &msg, sizeof(msg), 0);
233 if (size < sizeof(msg)) {
234 ret = -errno;
235 goto error;
236 }
237 ret = 0;
238
239 error:
240 return ret;
241 }
242
243 /*
244 * Communication call with the agent. Send the payload to the given socket. The
245 * header MUST be sent prior to this call.
246 *
247 * Return 0 on success or else a negative errno value of sendmsg() op.
248 */
249 static int send_payload(struct lttcomm_sock *sock, const void *data,
250 size_t size)
251 {
252 int ret;
253 ssize_t len;
254
255 assert(sock);
256 assert(data);
257
258 len = sock->ops->sendmsg(sock, data, size, 0);
259 if (len < size) {
260 ret = -errno;
261 goto error;
262 }
263 ret = 0;
264
265 error:
266 return ret;
267 }
268
269 /*
270 * Communication call with the agent. Receive reply from the agent using the
271 * given socket.
272 *
273 * Return 0 on success or else a negative errno value from recvmsg() op.
274 */
275 static int recv_reply(struct lttcomm_sock *sock, void *buf, size_t size)
276 {
277 int ret;
278 ssize_t len;
279
280 assert(sock);
281 assert(buf);
282
283 len = sock->ops->recvmsg(sock, buf, size, 0);
284 if (len < size) {
285 ret = -errno;
286 goto error;
287 }
288 ret = 0;
289
290 error:
291 return ret;
292 }
293
294 /*
295 * Internal event listing for a given app. Populate events.
296 *
297 * Return number of element in the list or else a negative LTTNG_ERR* code.
298 * On success, the caller is responsible for freeing the memory
299 * allocated for "events".
300 */
301 static ssize_t list_events(struct agent_app *app, struct lttng_event **events)
302 {
303 int ret, i, len = 0, offset = 0;
304 uint32_t nb_event;
305 size_t data_size;
306 uint32_t reply_ret_code;
307 struct lttng_event *tmp_events = NULL;
308 struct lttcomm_agent_list_reply *reply = NULL;
309 struct lttcomm_agent_list_reply_hdr reply_hdr;
310
311 assert(app);
312 assert(app->sock);
313 assert(events);
314
315 DBG2("Agent listing events for app pid: %d and socket %d", app->pid,
316 app->sock->fd);
317
318 ret = send_header(app->sock, 0, AGENT_CMD_LIST, 0);
319 if (ret < 0) {
320 goto error_io;
321 }
322
323 /* Get list header so we know how much we'll receive. */
324 ret = recv_reply(app->sock, &reply_hdr, sizeof(reply_hdr));
325 if (ret < 0) {
326 goto error_io;
327 }
328
329 reply_ret_code = be32toh(reply_hdr.ret_code);
330 log_reply_code(reply_ret_code);
331 switch (reply_ret_code) {
332 case AGENT_RET_CODE_SUCCESS:
333 data_size = be32toh(reply_hdr.data_size) + sizeof(*reply);
334 break;
335 default:
336 ret = LTTNG_ERR_UNK;
337 goto error;
338 }
339
340 reply = zmalloc(data_size);
341 if (!reply) {
342 ret = LTTNG_ERR_NOMEM;
343 goto error;
344 }
345
346 /* Get the list with the appropriate data size. */
347 ret = recv_reply(app->sock, reply, data_size);
348 if (ret < 0) {
349 goto error_io;
350 }
351
352 nb_event = be32toh(reply->nb_event);
353 tmp_events = zmalloc(sizeof(*tmp_events) * nb_event);
354 if (!tmp_events) {
355 ret = LTTNG_ERR_NOMEM;
356 goto error;
357 }
358
359 for (i = 0; i < nb_event; i++) {
360 offset += len;
361 if (lttng_strncpy(tmp_events[i].name, reply->payload + offset,
362 sizeof(tmp_events[i].name))) {
363 ret = LTTNG_ERR_INVALID;
364 goto error;
365 }
366 tmp_events[i].pid = app->pid;
367 tmp_events[i].enabled = -1;
368 len = strlen(reply->payload + offset) + 1;
369 }
370
371 *events = tmp_events;
372
373 free(reply);
374 return nb_event;
375
376 error_io:
377 ret = LTTNG_ERR_UST_LIST_FAIL;
378 error:
379 free(reply);
380 free(tmp_events);
381 return -ret;
382
383 }
384
385 /*
386 * Internal enable agent event on a agent application. This function
387 * communicates with the agent to enable a given event.
388 *
389 * Return LTTNG_OK on success or else a LTTNG_ERR* code.
390 */
391 static int enable_event(const struct agent_app *app, struct agent_event *event)
392 {
393 int ret;
394 char *bytes_to_send;
395 uint64_t data_size;
396 size_t filter_expression_length;
397 uint32_t reply_ret_code;
398 struct lttcomm_agent_enable_event msg;
399 struct lttcomm_agent_generic_reply reply;
400
401 assert(app);
402 assert(app->sock);
403 assert(event);
404
405 DBG2("Agent enabling event %s for app pid: %d and socket %d", event->name,
406 app->pid, app->sock->fd);
407
408 /*
409 * Calculate the payload's size, which is the fixed-size struct followed
410 * by the variable-length filter expression (+1 for the ending \0).
411 */
412 if (!event->filter_expression) {
413 filter_expression_length = 0;
414 } else {
415 filter_expression_length = strlen(event->filter_expression) + 1;
416 }
417 data_size = sizeof(msg) + filter_expression_length;
418
419 memset(&msg, 0, sizeof(msg));
420 msg.loglevel_value = htobe32(event->loglevel_value);
421 msg.loglevel_type = htobe32(event->loglevel_type);
422 if (lttng_strncpy(msg.name, event->name, sizeof(msg.name))) {
423 ret = LTTNG_ERR_INVALID;
424 goto error;
425 }
426 msg.filter_expression_length = htobe32(filter_expression_length);
427
428 ret = send_header(app->sock, data_size, AGENT_CMD_ENABLE, 0);
429 if (ret < 0) {
430 goto error_io;
431 }
432
433 bytes_to_send = zmalloc(data_size);
434 if (!bytes_to_send) {
435 ret = LTTNG_ERR_NOMEM;
436 goto error;
437 }
438
439 memcpy(bytes_to_send, &msg, sizeof(msg));
440 if (filter_expression_length > 0) {
441 memcpy(bytes_to_send + sizeof(msg), event->filter_expression,
442 filter_expression_length);
443 }
444
445 ret = send_payload(app->sock, bytes_to_send, data_size);
446 free(bytes_to_send);
447 if (ret < 0) {
448 goto error_io;
449 }
450
451 ret = recv_reply(app->sock, &reply, sizeof(reply));
452 if (ret < 0) {
453 goto error_io;
454 }
455
456 reply_ret_code = be32toh(reply.ret_code);
457 log_reply_code(reply_ret_code);
458 switch (reply_ret_code) {
459 case AGENT_RET_CODE_SUCCESS:
460 break;
461 case AGENT_RET_CODE_UNKNOWN_NAME:
462 ret = LTTNG_ERR_UST_EVENT_NOT_FOUND;
463 goto error;
464 default:
465 ret = LTTNG_ERR_UNK;
466 goto error;
467 }
468
469 return LTTNG_OK;
470
471 error_io:
472 ret = LTTNG_ERR_UST_ENABLE_FAIL;
473 error:
474 return ret;
475 }
476
477 /*
478 * Send Pascal-style string. Size is sent as a 32-bit big endian integer.
479 */
480 static
481 int send_pstring(struct lttcomm_sock *sock, const char *str, uint32_t len)
482 {
483 int ret;
484 uint32_t len_be;
485
486 len_be = htobe32(len);
487 ret = send_payload(sock, &len_be, sizeof(len_be));
488 if (ret) {
489 goto end;
490 }
491
492 ret = send_payload(sock, str, len);
493 if (ret) {
494 goto end;
495 }
496 end:
497 return ret;
498 }
499
500 /*
501 * Internal enable application context on an agent application. This function
502 * communicates with the agent to enable a given application context.
503 *
504 * Return LTTNG_OK on success or else a LTTNG_ERR* code.
505 */
506 static int app_context_op(const struct agent_app *app,
507 const struct agent_app_ctx *ctx, enum lttcomm_agent_command cmd)
508 {
509 int ret;
510 uint32_t reply_ret_code;
511 struct lttcomm_agent_generic_reply reply;
512 size_t app_ctx_provider_name_len, app_ctx_name_len, data_size;
513
514 assert(app);
515 assert(app->sock);
516 assert(ctx);
517 assert(cmd == AGENT_CMD_APP_CTX_ENABLE ||
518 cmd == AGENT_CMD_APP_CTX_DISABLE);
519
520 DBG2("Agent %s application %s:%s for app pid: %d and socket %d",
521 cmd == AGENT_CMD_APP_CTX_ENABLE ? "enabling" : "disabling",
522 ctx->provider_name, ctx->ctx_name,
523 app->pid, app->sock->fd);
524
525 /*
526 * Calculate the payload's size, which consists of the size (u32, BE)
527 * of the provider name, the NULL-terminated provider name string, the
528 * size (u32, BE) of the context name, followed by the NULL-terminated
529 * context name string.
530 */
531 app_ctx_provider_name_len = strlen(ctx->provider_name) + 1;
532 app_ctx_name_len = strlen(ctx->ctx_name) + 1;
533 data_size = sizeof(uint32_t) + app_ctx_provider_name_len +
534 sizeof(uint32_t) + app_ctx_name_len;
535
536 ret = send_header(app->sock, data_size, cmd, 0);
537 if (ret < 0) {
538 goto error_io;
539 }
540
541 if (app_ctx_provider_name_len > UINT32_MAX ||
542 app_ctx_name_len > UINT32_MAX) {
543 ERR("Application context name > MAX_UINT32");
544 ret = LTTNG_ERR_INVALID;
545 goto error;
546 }
547
548 ret = send_pstring(app->sock, ctx->provider_name,
549 (uint32_t) app_ctx_provider_name_len);
550 if (ret < 0) {
551 goto error_io;
552 }
553
554 ret = send_pstring(app->sock, ctx->ctx_name,
555 (uint32_t) app_ctx_name_len);
556 if (ret < 0) {
557 goto error_io;
558 }
559
560 ret = recv_reply(app->sock, &reply, sizeof(reply));
561 if (ret < 0) {
562 goto error_io;
563 }
564
565 reply_ret_code = be32toh(reply.ret_code);
566 log_reply_code(reply_ret_code);
567 switch (reply_ret_code) {
568 case AGENT_RET_CODE_SUCCESS:
569 break;
570 default:
571 ret = LTTNG_ERR_UNK;
572 goto error;
573 }
574
575 return LTTNG_OK;
576
577 error_io:
578 ret = LTTNG_ERR_UST_ENABLE_FAIL;
579 error:
580 return ret;
581 }
582
583 /*
584 * Internal disable agent event call on a agent application. This function
585 * communicates with the agent to disable a given event.
586 *
587 * Return LTTNG_OK on success or else a LTTNG_ERR* code.
588 */
589 static int disable_event(struct agent_app *app, struct agent_event *event)
590 {
591 int ret;
592 uint64_t data_size;
593 uint32_t reply_ret_code;
594 struct lttcomm_agent_disable_event msg;
595 struct lttcomm_agent_generic_reply reply;
596
597 assert(app);
598 assert(app->sock);
599 assert(event);
600
601 DBG2("Agent disabling event %s for app pid: %d and socket %d", event->name,
602 app->pid, app->sock->fd);
603
604 data_size = sizeof(msg);
605 memset(&msg, 0, sizeof(msg));
606 if (lttng_strncpy(msg.name, event->name, sizeof(msg.name))) {
607 ret = LTTNG_ERR_INVALID;
608 goto error;
609 }
610
611 ret = send_header(app->sock, data_size, AGENT_CMD_DISABLE, 0);
612 if (ret < 0) {
613 goto error_io;
614 }
615
616 ret = send_payload(app->sock, &msg, sizeof(msg));
617 if (ret < 0) {
618 goto error_io;
619 }
620
621 ret = recv_reply(app->sock, &reply, sizeof(reply));
622 if (ret < 0) {
623 goto error_io;
624 }
625
626 reply_ret_code = be32toh(reply.ret_code);
627 log_reply_code(reply_ret_code);
628 switch (reply_ret_code) {
629 case AGENT_RET_CODE_SUCCESS:
630 break;
631 case AGENT_RET_CODE_UNKNOWN_NAME:
632 ret = LTTNG_ERR_UST_EVENT_NOT_FOUND;
633 goto error;
634 default:
635 ret = LTTNG_ERR_UNK;
636 goto error;
637 }
638
639 return LTTNG_OK;
640
641 error_io:
642 ret = LTTNG_ERR_UST_DISABLE_FAIL;
643 error:
644 return ret;
645 }
646
647 /*
648 * Send back the registration DONE command to a given agent application.
649 *
650 * Return 0 on success or else a negative value.
651 */
652 int agent_send_registration_done(struct agent_app *app)
653 {
654 assert(app);
655 assert(app->sock);
656
657 DBG("Agent sending registration done to app socket %d", app->sock->fd);
658
659 return send_header(app->sock, 0, AGENT_CMD_REG_DONE, 0);
660 }
661
662 /*
663 * Enable agent event on every agent applications registered with the session
664 * daemon.
665 *
666 * Return LTTNG_OK on success or else a LTTNG_ERR* code.
667 */
668 int agent_enable_event(struct agent_event *event,
669 enum lttng_domain_type domain)
670 {
671 int ret;
672 struct agent_app *app;
673 struct lttng_ht_iter iter;
674
675 assert(event);
676
677 rcu_read_lock();
678
679 cds_lfht_for_each_entry(agent_apps_ht_by_sock->ht, &iter.iter, app,
680 node.node) {
681 if (app->domain != domain) {
682 continue;
683 }
684
685 /* Enable event on agent application through TCP socket. */
686 ret = enable_event(app, event);
687 if (ret != LTTNG_OK) {
688 goto error;
689 }
690 }
691
692 event->enabled_count++;
693 ret = LTTNG_OK;
694
695 error:
696 rcu_read_unlock();
697 return ret;
698 }
699
700 static
701 void destroy_app_ctx(struct agent_app_ctx *ctx)
702 {
703 free(ctx->provider_name);
704 free(ctx->ctx_name);
705 free(ctx);
706 }
707
708 static
709 struct agent_app_ctx *create_app_ctx(const struct lttng_event_context *ctx)
710 {
711 struct agent_app_ctx *agent_ctx = NULL;
712
713 if (!ctx) {
714 goto end;
715 }
716
717 assert(ctx->ctx == LTTNG_EVENT_CONTEXT_APP_CONTEXT);
718 agent_ctx = zmalloc(sizeof(*ctx));
719 if (!agent_ctx) {
720 goto end;
721 }
722
723 agent_ctx->provider_name = strdup(ctx->u.app_ctx.provider_name);
724 agent_ctx->ctx_name = strdup(ctx->u.app_ctx.ctx_name);
725 if (!agent_ctx->provider_name || !agent_ctx->ctx_name) {
726 destroy_app_ctx(agent_ctx);
727 agent_ctx = NULL;
728 }
729 end:
730 return agent_ctx;
731 }
732
733 /*
734 * Enable agent context on every agent applications registered with the session
735 * daemon.
736 *
737 * Return LTTNG_OK on success or else a LTTNG_ERR* code.
738 */
739 int agent_enable_context(const struct lttng_event_context *ctx,
740 enum lttng_domain_type domain)
741 {
742 int ret;
743 struct agent_app *app;
744 struct lttng_ht_iter iter;
745
746 assert(ctx);
747 if (ctx->ctx != LTTNG_EVENT_CONTEXT_APP_CONTEXT) {
748 ret = LTTNG_ERR_INVALID;
749 goto error;
750 }
751
752 rcu_read_lock();
753
754 cds_lfht_for_each_entry(agent_apps_ht_by_sock->ht, &iter.iter, app,
755 node.node) {
756 struct agent_app_ctx *agent_ctx;
757
758 if (app->domain != domain) {
759 continue;
760 }
761
762 agent_ctx = create_app_ctx(ctx);
763 if (!agent_ctx) {
764 ret = LTTNG_ERR_NOMEM;
765 goto error_unlock;
766 }
767
768 /* Enable event on agent application through TCP socket. */
769 ret = app_context_op(app, agent_ctx, AGENT_CMD_APP_CTX_ENABLE);
770 destroy_app_ctx(agent_ctx);
771 if (ret != LTTNG_OK) {
772 goto error_unlock;
773 }
774 }
775
776 ret = LTTNG_OK;
777
778 error_unlock:
779 rcu_read_unlock();
780 error:
781 return ret;
782 }
783
784 /*
785 * Disable agent event on every agent application registered with the session
786 * daemon.
787 *
788 * Return LTTNG_OK on success or else a LTTNG_ERR* code.
789 */
790 int agent_disable_event(struct agent_event *event,
791 enum lttng_domain_type domain)
792 {
793 int ret = LTTNG_OK;
794 struct agent_app *app;
795 struct lttng_ht_iter iter;
796
797 assert(event);
798 if (!AGENT_EVENT_IS_ENABLED(event)) {
799 goto end;
800 }
801
802 if (--event->enabled_count != 0) {
803 /*
804 * Agent event still enabled. Disable the agent event only when
805 * all "users" have disabled it (event notifiers, event rules,
806 * etc.).
807 */
808 ret = LTTNG_OK;
809 goto end;
810 }
811
812 rcu_read_lock();
813
814 cds_lfht_for_each_entry(agent_apps_ht_by_sock->ht, &iter.iter, app,
815 node.node) {
816 if (app->domain != domain) {
817 continue;
818 }
819
820 /* Enable event on agent application through TCP socket. */
821 ret = disable_event(app, event);
822 if (ret != LTTNG_OK) {
823 goto error;
824 }
825 }
826
827 /* event->enabled_count is now 0. */
828 assert(!AGENT_EVENT_IS_ENABLED(event));
829
830 error:
831 rcu_read_unlock();
832 end:
833 return ret;
834 }
835
836 /*
837 * Disable agent context on every agent application registered with the session
838 * daemon.
839 *
840 * Return LTTNG_OK on success or else a LTTNG_ERR* code.
841 */
842 static int disable_context(struct agent_app_ctx *ctx,
843 enum lttng_domain_type domain)
844 {
845 int ret = LTTNG_OK;
846 struct agent_app *app;
847 struct lttng_ht_iter iter;
848
849 assert(ctx);
850
851 rcu_read_lock();
852 DBG2("Disabling agent application context %s:%s",
853 ctx->provider_name, ctx->ctx_name);
854 cds_lfht_for_each_entry(agent_apps_ht_by_sock->ht, &iter.iter, app,
855 node.node) {
856 if (app->domain != domain) {
857 continue;
858 }
859
860 ret = app_context_op(app, ctx, AGENT_CMD_APP_CTX_DISABLE);
861 if (ret != LTTNG_OK) {
862 goto end;
863 }
864 }
865 end:
866 rcu_read_unlock();
867 return ret;
868 }
869
870 /*
871 * Ask every agent for the list of possible event. Events is allocated with the
872 * events of every agent application.
873 *
874 * Return the number of events or else a negative value.
875 */
876 int agent_list_events(struct lttng_event **events,
877 enum lttng_domain_type domain)
878 {
879 int ret;
880 size_t nbmem, count = 0;
881 struct agent_app *app;
882 struct lttng_event *tmp_events = NULL;
883 struct lttng_ht_iter iter;
884
885 assert(events);
886
887 DBG2("Agent listing events for domain %d", domain);
888
889 nbmem = UST_APP_EVENT_LIST_SIZE;
890 tmp_events = zmalloc(nbmem * sizeof(*tmp_events));
891 if (!tmp_events) {
892 PERROR("zmalloc agent list events");
893 ret = -ENOMEM;
894 goto error;
895 }
896
897 rcu_read_lock();
898 cds_lfht_for_each_entry(agent_apps_ht_by_sock->ht, &iter.iter, app,
899 node.node) {
900 ssize_t nb_ev;
901 struct lttng_event *agent_events;
902
903 /* Skip domain not asked by the list. */
904 if (app->domain != domain) {
905 continue;
906 }
907
908 nb_ev = list_events(app, &agent_events);
909 if (nb_ev < 0) {
910 ret = nb_ev;
911 goto error_unlock;
912 }
913
914 if (count + nb_ev > nbmem) {
915 /* In case the realloc fails, we free the memory */
916 struct lttng_event *new_tmp_events;
917 size_t new_nbmem;
918
919 new_nbmem = max_t(size_t, count + nb_ev, nbmem << 1);
920 DBG2("Reallocating agent event list from %zu to %zu entries",
921 nbmem, new_nbmem);
922 new_tmp_events = realloc(tmp_events,
923 new_nbmem * sizeof(*new_tmp_events));
924 if (!new_tmp_events) {
925 PERROR("realloc agent events");
926 ret = -ENOMEM;
927 free(agent_events);
928 goto error_unlock;
929 }
930 /* Zero the new memory */
931 memset(new_tmp_events + nbmem, 0,
932 (new_nbmem - nbmem) * sizeof(*new_tmp_events));
933 nbmem = new_nbmem;
934 tmp_events = new_tmp_events;
935 }
936 memcpy(tmp_events + count, agent_events,
937 nb_ev * sizeof(*tmp_events));
938 free(agent_events);
939 count += nb_ev;
940 }
941 rcu_read_unlock();
942
943 ret = count;
944 *events = tmp_events;
945 return ret;
946
947 error_unlock:
948 rcu_read_unlock();
949 error:
950 free(tmp_events);
951 return ret;
952 }
953
954 /*
955 * Create a agent app object using the given PID.
956 *
957 * Return newly allocated object or else NULL on error.
958 */
959 struct agent_app *agent_create_app(pid_t pid, enum lttng_domain_type domain,
960 struct lttcomm_sock *sock)
961 {
962 struct agent_app *app;
963
964 assert(sock);
965
966 app = zmalloc(sizeof(*app));
967 if (!app) {
968 PERROR("Failed to allocate agent application instance");
969 goto error;
970 }
971
972 app->pid = pid;
973 app->domain = domain;
974 app->sock = sock;
975 lttng_ht_node_init_ulong(&app->node, (unsigned long) app->sock->fd);
976
977 error:
978 return app;
979 }
980
981 /*
982 * Lookup agent app by socket in the global hash table.
983 *
984 * RCU read side lock MUST be acquired.
985 *
986 * Return object if found else NULL.
987 */
988 struct agent_app *agent_find_app_by_sock(int sock)
989 {
990 struct lttng_ht_node_ulong *node;
991 struct lttng_ht_iter iter;
992 struct agent_app *app;
993
994 assert(sock >= 0);
995
996 lttng_ht_lookup(agent_apps_ht_by_sock, (void *)((unsigned long) sock), &iter);
997 node = lttng_ht_iter_get_node_ulong(&iter);
998 if (node == NULL) {
999 goto error;
1000 }
1001 app = caa_container_of(node, struct agent_app, node);
1002
1003 DBG3("Agent app pid %d found by sock %d.", app->pid, sock);
1004 return app;
1005
1006 error:
1007 DBG3("Agent app NOT found by sock %d.", sock);
1008 return NULL;
1009 }
1010
1011 /*
1012 * Add agent application object to the global hash table.
1013 */
1014 void agent_add_app(struct agent_app *app)
1015 {
1016 assert(app);
1017
1018 DBG3("Agent adding app sock: %d and pid: %d to ht", app->sock->fd, app->pid);
1019 lttng_ht_add_unique_ulong(agent_apps_ht_by_sock, &app->node);
1020 }
1021
1022 /*
1023 * Delete agent application from the global hash table.
1024 *
1025 * rcu_read_lock() must be held by the caller.
1026 */
1027 void agent_delete_app(struct agent_app *app)
1028 {
1029 int ret;
1030 struct lttng_ht_iter iter;
1031
1032 assert(app);
1033
1034 DBG3("Agent deleting app pid: %d and sock: %d", app->pid, app->sock->fd);
1035
1036 iter.iter.node = &app->node.node;
1037 ret = lttng_ht_del(agent_apps_ht_by_sock, &iter);
1038 assert(!ret);
1039 }
1040
1041 /*
1042 * Destroy an agent application object by detaching it from its corresponding
1043 * UST app if one is connected by closing the socket. Finally, perform a
1044 * delayed memory reclaim.
1045 */
1046 void agent_destroy_app(struct agent_app *app)
1047 {
1048 assert(app);
1049
1050 if (app->sock) {
1051 app->sock->ops->close(app->sock);
1052 lttcomm_destroy_sock(app->sock);
1053 }
1054
1055 call_rcu(&app->node.head, destroy_app_agent_rcu);
1056 }
1057
1058 /*
1059 * Initialize an already allocated agent object.
1060 *
1061 * Return 0 on success or else a negative errno value.
1062 */
1063 int agent_init(struct agent *agt)
1064 {
1065 int ret;
1066
1067 assert(agt);
1068
1069 agt->events = lttng_ht_new(0, LTTNG_HT_TYPE_STRING);
1070 if (!agt->events) {
1071 ret = -ENOMEM;
1072 goto error;
1073 }
1074 lttng_ht_node_init_u64(&agt->node, agt->domain);
1075
1076 CDS_INIT_LIST_HEAD(&agt->app_ctx_list);
1077 return 0;
1078
1079 error:
1080 return ret;
1081 }
1082
1083 /*
1084 * Add agent object to the given hash table.
1085 */
1086 void agent_add(struct agent *agt, struct lttng_ht *ht)
1087 {
1088 assert(agt);
1089 assert(ht);
1090
1091 DBG3("Agent adding from domain %d", agt->domain);
1092
1093 lttng_ht_add_unique_u64(ht, &agt->node);
1094 }
1095
1096 /*
1097 * Create an agent object for the given domain.
1098 *
1099 * Return the allocated agent or NULL on error.
1100 */
1101 struct agent *agent_create(enum lttng_domain_type domain)
1102 {
1103 int ret;
1104 struct agent *agt;
1105
1106 agt = zmalloc(sizeof(struct agent));
1107 if (!agt) {
1108 goto error;
1109 }
1110 agt->domain = domain;
1111
1112 ret = agent_init(agt);
1113 if (ret < 0) {
1114 free(agt);
1115 agt = NULL;
1116 goto error;
1117 }
1118
1119 error:
1120 return agt;
1121 }
1122
1123 /*
1124 * Create a newly allocated agent event data structure.
1125 * Ownership of filter_expression is taken.
1126 *
1127 * Return a new object else NULL on error.
1128 */
1129 struct agent_event *agent_create_event(const char *name,
1130 enum lttng_loglevel_type loglevel_type, int loglevel_value,
1131 struct lttng_filter_bytecode *filter, char *filter_expression)
1132 {
1133 struct agent_event *event = NULL;
1134
1135 DBG3("Agent create new event with name %s, loglevel type %d, \
1136 loglevel value %d and filter %s",
1137 name, loglevel_type, loglevel_value,
1138 filter_expression ? filter_expression : "NULL");
1139
1140 if (!name) {
1141 ERR("Failed to create agent event; no name provided.");
1142 goto error;
1143 }
1144
1145 event = zmalloc(sizeof(*event));
1146 if (!event) {
1147 goto error;
1148 }
1149
1150 strncpy(event->name, name, sizeof(event->name));
1151 event->name[sizeof(event->name) - 1] = '\0';
1152 lttng_ht_node_init_str(&event->node, event->name);
1153
1154 event->loglevel_value = loglevel_value;
1155 event->loglevel_type = loglevel_type;
1156 event->filter = filter;
1157 event->filter_expression = filter_expression;
1158 error:
1159 return event;
1160 }
1161
1162 /*
1163 * Unique add of a agent event to an agent object.
1164 */
1165 void agent_add_event(struct agent_event *event, struct agent *agt)
1166 {
1167 assert(event);
1168 assert(agt);
1169 assert(agt->events);
1170
1171 DBG3("Agent adding event %s", event->name);
1172 add_unique_agent_event(agt->events, event);
1173 agt->being_used = 1;
1174 }
1175
1176 /*
1177 * Unique add of a agent context to an agent object.
1178 */
1179 int agent_add_context(const struct lttng_event_context *ctx, struct agent *agt)
1180 {
1181 int ret = LTTNG_OK;
1182 struct agent_app_ctx *agent_ctx = NULL;
1183
1184 assert(ctx);
1185 assert(agt);
1186 assert(agt->events);
1187 assert(ctx->ctx == LTTNG_EVENT_CONTEXT_APP_CONTEXT);
1188
1189 agent_ctx = create_app_ctx(ctx);
1190 if (!agent_ctx) {
1191 ret = LTTNG_ERR_NOMEM;
1192 goto end;
1193 }
1194
1195 DBG3("Agent adding context %s:%s", ctx->u.app_ctx.provider_name,
1196 ctx->u.app_ctx.ctx_name);
1197 cds_list_add_tail_rcu(&agent_ctx->list_node, &agt->app_ctx_list);
1198 end:
1199 return ret;
1200 }
1201
1202 /*
1203 * Find multiple agent events sharing the given name.
1204 *
1205 * RCU read side lock MUST be acquired. It must be held for the
1206 * duration of the iteration.
1207 *
1208 * Sets the given iterator.
1209 */
1210 void agent_find_events_by_name(const char *name, struct agent *agt,
1211 struct lttng_ht_iter* iter)
1212 {
1213 struct lttng_ht *ht;
1214 struct agent_ht_key key;
1215
1216 assert(name);
1217 assert(agt);
1218 assert(agt->events);
1219 assert(iter);
1220
1221 ht = agt->events;
1222 key.name = name;
1223
1224 cds_lfht_lookup(ht->ht, ht->hash_fct((void *) name, lttng_ht_seed),
1225 ht_match_event_by_name, &key, &iter->iter);
1226 }
1227
1228 /*
1229 * Find the agent event matching a trigger.
1230 *
1231 * RCU read side lock MUST be acquired. It must be held for as long as
1232 * the returned agent_event is used.
1233 *
1234 * Return object if found else NULL.
1235 */
1236 struct agent_event *agent_find_event_by_trigger(
1237 const struct lttng_trigger *trigger, struct agent *agt)
1238 {
1239 enum lttng_condition_status c_status;
1240 enum lttng_event_rule_status er_status;
1241 enum lttng_domain_type domain;
1242 const struct lttng_condition *condition;
1243 const struct lttng_event_rule *rule;
1244 const char *name;
1245 const char *filter_expression;
1246 /* Unused when loglevel_type is 'ALL'. */
1247 int loglevel_value = 0;
1248 enum lttng_loglevel_type loglevel_type;
1249
1250 assert(agt);
1251 assert(agt->events);
1252
1253 condition = lttng_trigger_get_const_condition(trigger);
1254
1255 assert(lttng_condition_get_type(condition) ==
1256 LTTNG_CONDITION_TYPE_EVENT_RULE_HIT);
1257
1258 c_status = lttng_condition_event_rule_get_rule(condition, &rule);
1259 assert(c_status == LTTNG_CONDITION_STATUS_OK);
1260
1261 assert(lttng_event_rule_get_type(rule) ==
1262 LTTNG_EVENT_RULE_TYPE_TRACEPOINT);
1263
1264 domain = lttng_event_rule_get_domain_type(rule);
1265 assert(domain == LTTNG_DOMAIN_JUL || domain == LTTNG_DOMAIN_LOG4J ||
1266 domain == LTTNG_DOMAIN_PYTHON);
1267
1268 /* Get the event's pattern ('name' in the legacy terminology). */
1269 er_status = lttng_event_rule_tracepoint_get_pattern(rule, &name);
1270 assert(er_status == LTTNG_EVENT_RULE_STATUS_OK);
1271
1272 /* Get the internal filter expression. */
1273 filter_expression = lttng_event_rule_get_filter(rule);
1274
1275 er_status = lttng_event_rule_tracepoint_get_log_level_type(
1276 rule, &loglevel_type);
1277 assert(er_status == LTTNG_EVENT_RULE_STATUS_OK);
1278
1279 if (loglevel_type != LTTNG_EVENT_LOGLEVEL_ALL) {
1280 er_status = lttng_event_rule_tracepoint_get_log_level(
1281 rule, &loglevel_value);
1282 assert(er_status == LTTNG_EVENT_RULE_STATUS_OK);
1283 }
1284
1285 return agent_find_event(name, loglevel_type, loglevel_value,
1286 filter_expression, agt);
1287 }
1288
1289 /*
1290 * Get the next agent event duplicate by name. This should be called
1291 * after a call to agent_find_events_by_name() to iterate on events.
1292 *
1293 * The RCU read lock must be held during the iteration and for as long
1294 * as the object the iterator points to remains in use.
1295 */
1296 void agent_event_next_duplicate(const char *name,
1297 struct agent *agt, struct lttng_ht_iter* iter)
1298 {
1299 struct agent_ht_key key;
1300
1301 key.name = name;
1302
1303 cds_lfht_next_duplicate(agt->events->ht, ht_match_event_by_name,
1304 &key, &iter->iter);
1305 }
1306
1307 /*
1308 * Find a agent event in the given agent using name, loglevel and filter.
1309 *
1310 * RCU read side lock MUST be acquired. It must be kept for as long as
1311 * the returned agent_event is used.
1312 *
1313 * Return object if found else NULL.
1314 */
1315 struct agent_event *agent_find_event(const char *name,
1316 enum lttng_loglevel_type loglevel_type,
1317 int loglevel_value,
1318 const char *filter_expression,
1319 struct agent *agt)
1320 {
1321 struct lttng_ht_node_str *node;
1322 struct lttng_ht_iter iter;
1323 struct lttng_ht *ht;
1324 struct agent_ht_key key;
1325
1326 assert(name);
1327 assert(agt);
1328 assert(agt->events);
1329
1330 ht = agt->events;
1331 key.name = name;
1332 key.loglevel_value = loglevel_value;
1333 key.loglevel_type = loglevel_type;
1334 key.filter_expression = filter_expression;
1335
1336 cds_lfht_lookup(ht->ht, ht->hash_fct((void *) name, lttng_ht_seed),
1337 ht_match_event, &key, &iter.iter);
1338 node = lttng_ht_iter_get_node_str(&iter);
1339 if (node == NULL) {
1340 goto error;
1341 }
1342
1343 DBG3("Agent event found %s.", name);
1344 return caa_container_of(node, struct agent_event, node);
1345
1346 error:
1347 DBG3("Agent event NOT found %s.", name);
1348 return NULL;
1349 }
1350
1351 /*
1352 * Free given agent event. This event must not be globally visible at this
1353 * point (only expected to be used on failure just after event creation). After
1354 * this call, the pointer is not usable anymore.
1355 */
1356 void agent_destroy_event(struct agent_event *event)
1357 {
1358 assert(event);
1359
1360 free(event->filter);
1361 free(event->filter_expression);
1362 free(event->exclusion);
1363 free(event);
1364 }
1365
1366 static
1367 void destroy_app_ctx_rcu(struct rcu_head *head)
1368 {
1369 struct agent_app_ctx *ctx =
1370 caa_container_of(head, struct agent_app_ctx, rcu_node);
1371
1372 destroy_app_ctx(ctx);
1373 }
1374
1375 /*
1376 * Destroy an agent completely.
1377 */
1378 void agent_destroy(struct agent *agt)
1379 {
1380 struct lttng_ht_node_str *node;
1381 struct lttng_ht_iter iter;
1382 struct agent_app_ctx *ctx;
1383
1384 assert(agt);
1385
1386 DBG3("Agent destroy");
1387
1388 rcu_read_lock();
1389 cds_lfht_for_each_entry(agt->events->ht, &iter.iter, node, node) {
1390 int ret;
1391 struct agent_event *event;
1392
1393 /*
1394 * When destroying an event, we have to try to disable it on the
1395 * agent side so the event stops generating data. The return
1396 * value is not important since we have to continue anyway
1397 * destroying the object.
1398 */
1399 event = caa_container_of(node, struct agent_event, node);
1400 (void) agent_disable_event(event, agt->domain);
1401
1402 ret = lttng_ht_del(agt->events, &iter);
1403 assert(!ret);
1404 call_rcu(&node->head, destroy_event_agent_rcu);
1405 }
1406
1407 cds_list_for_each_entry_rcu(ctx, &agt->app_ctx_list, list_node) {
1408 (void) disable_context(ctx, agt->domain);
1409 cds_list_del(&ctx->list_node);
1410 call_rcu(&ctx->rcu_node, destroy_app_ctx_rcu);
1411 }
1412 rcu_read_unlock();
1413 ht_cleanup_push(agt->events);
1414 free(agt);
1415 }
1416
1417 /*
1418 * Allocate agent_apps_ht_by_sock.
1419 */
1420 int agent_app_ht_alloc(void)
1421 {
1422 agent_apps_ht_by_sock = lttng_ht_new(0, LTTNG_HT_TYPE_ULONG);
1423 return agent_apps_ht_by_sock ? 0 : -1;
1424 }
1425
1426 /*
1427 * Destroy a agent application by socket.
1428 */
1429 void agent_destroy_app_by_sock(int sock)
1430 {
1431 struct agent_app *app;
1432
1433 assert(sock >= 0);
1434
1435 /*
1436 * Not finding an application is a very important error that should NEVER
1437 * happen. The hash table deletion is ONLY done through this call when the
1438 * main sessiond thread is torn down.
1439 */
1440 rcu_read_lock();
1441 app = agent_find_app_by_sock(sock);
1442 assert(app);
1443
1444 /* RCU read side lock is assumed to be held by this function. */
1445 agent_delete_app(app);
1446
1447 /* The application is freed in a RCU call but the socket is closed here. */
1448 agent_destroy_app(app);
1449 rcu_read_unlock();
1450 }
1451
1452 /*
1453 * Clean-up the agent app hash table and destroy it.
1454 */
1455 void agent_app_ht_clean(void)
1456 {
1457 struct lttng_ht_node_ulong *node;
1458 struct lttng_ht_iter iter;
1459
1460 if (!agent_apps_ht_by_sock) {
1461 return;
1462 }
1463 rcu_read_lock();
1464 cds_lfht_for_each_entry(agent_apps_ht_by_sock->ht, &iter.iter, node, node) {
1465 struct agent_app *app;
1466
1467 app = caa_container_of(node, struct agent_app, node);
1468 agent_destroy_app_by_sock(app->sock->fd);
1469 }
1470 rcu_read_unlock();
1471
1472 lttng_ht_destroy(agent_apps_ht_by_sock);
1473 }
1474
1475 /*
1476 * Update a agent application (given socket) using the given agent.
1477 *
1478 * Note that this function is most likely to be used with a tracing session
1479 * thus the caller should make sure to hold the appropriate lock(s).
1480 */
1481 void agent_update(const struct agent *agt, const struct agent_app *app)
1482 {
1483 int ret;
1484 struct agent_event *event;
1485 struct lttng_ht_iter iter;
1486 struct agent_app_ctx *ctx;
1487
1488 assert(agt);
1489 assert(app);
1490
1491 DBG("Agent updating app: pid = %ld", (long) app->pid);
1492
1493 rcu_read_lock();
1494 /*
1495 * We are in the registration path thus if the application is gone,
1496 * there is a serious code flow error.
1497 */
1498
1499 cds_lfht_for_each_entry(agt->events->ht, &iter.iter, event, node.node) {
1500 /* Skip event if disabled. */
1501 if (!AGENT_EVENT_IS_ENABLED(event)) {
1502 continue;
1503 }
1504
1505 ret = enable_event(app, event);
1506 if (ret != LTTNG_OK) {
1507 DBG2("Agent update unable to enable event %s on app pid: %d sock %d",
1508 event->name, app->pid, app->sock->fd);
1509 /* Let's try the others here and don't assume the app is dead. */
1510 continue;
1511 }
1512 }
1513
1514 cds_list_for_each_entry_rcu(ctx, &agt->app_ctx_list, list_node) {
1515 ret = app_context_op(app, ctx, AGENT_CMD_APP_CTX_ENABLE);
1516 if (ret != LTTNG_OK) {
1517 DBG2("Agent update unable to add application context %s:%s on app pid: %d sock %d",
1518 ctx->provider_name, ctx->ctx_name,
1519 app->pid, app->sock->fd);
1520 continue;
1521 }
1522 }
1523
1524 rcu_read_unlock();
1525 }
1526
1527 /*
1528 * Allocate the per-event notifier domain agent hash table. It is lazily
1529 * populated as domains are used.
1530 */
1531 int agent_by_event_notifier_domain_ht_create(void)
1532 {
1533 trigger_agents_ht_by_domain = lttng_ht_new(0, LTTNG_HT_TYPE_U64);
1534 return trigger_agents_ht_by_domain ? 0 : -1;
1535 }
1536
1537 /*
1538 * Clean-up the per-event notifier domain agent hash table and destroy it.
1539 */
1540 void agent_by_event_notifier_domain_ht_destroy(void)
1541 {
1542 struct lttng_ht_node_u64 *node;
1543 struct lttng_ht_iter iter;
1544
1545 if (!trigger_agents_ht_by_domain) {
1546 return;
1547 }
1548
1549 rcu_read_lock();
1550 cds_lfht_for_each_entry (trigger_agents_ht_by_domain->ht, &iter.iter,
1551 node, node) {
1552 struct agent *agent =
1553 caa_container_of(node, struct agent, node);
1554 const int ret = lttng_ht_del(
1555 trigger_agents_ht_by_domain, &iter);
1556
1557 assert(ret == 0);
1558 agent_destroy(agent);
1559 }
1560
1561 rcu_read_unlock();
1562 lttng_ht_destroy(trigger_agents_ht_by_domain);
1563 }
1564
1565 struct agent *agent_find_by_event_notifier_domain(
1566 enum lttng_domain_type domain_type)
1567 {
1568 struct agent *agt = NULL;
1569 struct lttng_ht_node_u64 *node;
1570 struct lttng_ht_iter iter;
1571 const uint64_t key = (uint64_t) domain_type;
1572
1573 assert(trigger_agents_ht_by_domain);
1574
1575 DBG3("Per-event notifier domain agent lookup for domain '%s'",
1576 lttng_domain_type_str(domain_type));
1577
1578 lttng_ht_lookup(trigger_agents_ht_by_domain, &key, &iter);
1579 node = lttng_ht_iter_get_node_u64(&iter);
1580 if (!node) {
1581 goto end;
1582 }
1583
1584 agt = caa_container_of(node, struct agent, node);
1585
1586 end:
1587 return agt;
1588 }
This page took 0.100907 seconds and 4 git commands to generate.