2 * Copyright (C) 2011 - Julien Desfossez <julien.desfossez@polymtl.ca>
3 * Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; either version 2
8 * of the License, or (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
32 #include <sys/socket.h>
34 #include <sys/types.h>
35 #include <urcu/list.h>
40 #include "libkernelctl.h"
41 #include "liblttsessiondcomm.h"
42 #include "kconsumerd.h"
44 /* Init the list of FDs */
45 static struct ltt_kconsumerd_fd_list kconsumerd_fd_list
= {
46 .head
= CDS_LIST_HEAD_INIT(kconsumerd_fd_list
.head
),
49 /* Number of element for the list below. */
50 static unsigned int fds_count
;
52 /* If the local array of FDs needs update in the poll function */
53 static unsigned int update_fd_array
= 1;
55 /* lock the fd array and structures */
56 static pthread_mutex_t kconsumerd_lock_fds
;
58 /* the two threads (receive fd and poll) */
59 static pthread_t threads
[2];
61 /* communication with splice */
62 static int thread_pipe
[2];
64 /* pipe to wake the poll thread when necessary */
65 static int poll_pipe
[2];
67 /* socket to communicate errors with sessiond */
68 static int error_socket
= -1;
70 /* to count the number of time the user pressed ctrl+c */
71 static int sigintcount
= 0;
73 /* flag to inform the polling thread to quit when all fd hung up */
76 /* Argument variables */
79 static int opt_daemon
;
80 static const char *progname
;
81 static char command_sock_path
[PATH_MAX
]; /* Global command socket path */
82 static char error_sock_path
[PATH_MAX
]; /* Global error path */
87 * Remove a fd from the global list protected by a mutex
89 static void del_fd(struct ltt_kconsumerd_fd
*lcf
)
91 DBG("Removing %d", lcf
->consumerd_fd
);
92 pthread_mutex_lock(&kconsumerd_lock_fds
);
93 cds_list_del(&lcf
->list
);
96 DBG("Removed ltt_kconsumerd_fd");
99 close(lcf
->consumerd_fd
);
104 pthread_mutex_unlock(&kconsumerd_lock_fds
);
110 * Cleanup the daemon's socket on exit
112 static void cleanup()
114 struct ltt_kconsumerd_fd
*iter
;
116 /* remove the socket file */
117 unlink(command_sock_path
);
119 /* unblock the threads */
120 WARN("Terminating the threads before exiting");
121 pthread_cancel(threads
[0]);
122 pthread_cancel(threads
[1]);
124 /* close all outfd */
125 cds_list_for_each_entry(iter
, &kconsumerd_fd_list
.head
, list
) {
133 * send return code to ltt-sessiond
135 static int send_error(enum lttcomm_return_code cmd
)
137 if (error_socket
> 0) {
138 return lttcomm_send_unix_sock(error_socket
, &cmd
,
139 sizeof(enum lttcomm_sessiond_command
));
148 * Add a fd to the global list protected by a mutex
150 static int add_fd(struct lttcomm_kconsumerd_msg
*buf
, int consumerd_fd
)
152 struct ltt_kconsumerd_fd
*tmp_fd
;
155 tmp_fd
= malloc(sizeof(struct ltt_kconsumerd_fd
));
156 tmp_fd
->sessiond_fd
= buf
->fd
;
157 tmp_fd
->consumerd_fd
= consumerd_fd
;
158 tmp_fd
->state
= buf
->state
;
159 tmp_fd
->max_sb_size
= buf
->max_sb_size
;
160 strncpy(tmp_fd
->path_name
, buf
->path_name
, PATH_MAX
);
162 /* Opening the tracefile in write mode */
163 DBG("Opening %s for writing", tmp_fd
->path_name
);
164 ret
= open(tmp_fd
->path_name
,
165 O_WRONLY
|O_CREAT
|O_TRUNC
, S_IRWXU
|S_IRWXG
|S_IRWXO
);
167 ERR("Opening %s", tmp_fd
->path_name
);
171 tmp_fd
->out_fd
= ret
;
172 tmp_fd
->out_fd_offset
= 0;
174 DBG("Adding %s (%d, %d, %d)", tmp_fd
->path_name
,
175 tmp_fd
->sessiond_fd
, tmp_fd
->consumerd_fd
, tmp_fd
->out_fd
);
177 pthread_mutex_lock(&kconsumerd_lock_fds
);
178 cds_list_add(&tmp_fd
->list
, &kconsumerd_fd_list
.head
);
180 pthread_mutex_unlock(&kconsumerd_lock_fds
);
190 * Signal handler for the daemon
192 static void sighandler(int sig
)
194 if (sig
== SIGINT
&& sigintcount
++ == 0) {
195 DBG("ignoring first SIGINT");
207 * Setup signal handler for :
208 * SIGINT, SIGTERM, SIGPIPE
210 static int set_signal_handler(void)
216 if ((ret
= sigemptyset(&sigset
)) < 0) {
217 perror("sigemptyset");
221 sa
.sa_handler
= sighandler
;
224 if ((ret
= sigaction(SIGTERM
, &sa
, NULL
)) < 0) {
229 if ((ret
= sigaction(SIGINT
, &sa
, NULL
)) < 0) {
234 if ((ret
= sigaction(SIGPIPE
, &sa
, NULL
)) < 0) {
245 * Splice the data from the ring buffer to the tracefile.
246 * Returns the number of bytes spliced
248 static int on_read_subbuffer(struct ltt_kconsumerd_fd
*kconsumerd_fd
,
253 off_t orig_offset
= kconsumerd_fd
->out_fd_offset
;
254 int fd
= kconsumerd_fd
->consumerd_fd
;
255 int outfd
= kconsumerd_fd
->out_fd
;
258 DBG("splice chan to pipe offset %lu (fd : %d)",
259 (unsigned long)offset
, fd
);
260 ret
= splice(fd
, &offset
, thread_pipe
[1], NULL
, len
,
261 SPLICE_F_MOVE
| SPLICE_F_MORE
);
262 DBG("splice chan to pipe ret %ld", ret
);
265 perror("Error in relay splice");
269 ret
= splice(thread_pipe
[0], NULL
, outfd
, NULL
, ret
,
270 SPLICE_F_MOVE
| SPLICE_F_MORE
);
271 DBG("splice pipe to file %ld", ret
);
274 perror("Error in file splice");
280 /* This won't block, but will start writeout asynchronously */
281 sync_file_range(outfd
, kconsumerd_fd
->out_fd_offset
, ret
,
282 SYNC_FILE_RANGE_WRITE
);
283 kconsumerd_fd
->out_fd_offset
+= ret
;
287 * This does a blocking write-and-wait on any page that belongs to the
288 * subbuffer prior to the one we just wrote.
289 * Don't care about error values, as these are just hints and ways to
290 * limit the amount of page cache used.
292 if (orig_offset
>= kconsumerd_fd
->max_sb_size
) {
293 sync_file_range(outfd
, orig_offset
- kconsumerd_fd
->max_sb_size
,
294 kconsumerd_fd
->max_sb_size
,
295 SYNC_FILE_RANGE_WAIT_BEFORE
296 | SYNC_FILE_RANGE_WRITE
297 | SYNC_FILE_RANGE_WAIT_AFTER
);
299 * Give hints to the kernel about how we access the file:
300 * POSIX_FADV_DONTNEED : we won't re-access data in a near
301 * future after we write it.
302 * We need to call fadvise again after the file grows because
303 * the kernel does not seem to apply fadvise to non-existing
305 * Call fadvise _after_ having waited for the page writeback to
306 * complete because the dirty page writeback semantic is not
307 * well defined. So it can be expected to lead to lower
308 * throughput in streaming.
310 posix_fadvise(outfd
, orig_offset
- kconsumerd_fd
->max_sb_size
,
311 kconsumerd_fd
->max_sb_size
, POSIX_FADV_DONTNEED
);
316 /* send the appropriate error description to sessiond */
319 send_error(KCONSUMERD_SPLICE_EBADF
);
322 send_error(KCONSUMERD_SPLICE_EINVAL
);
325 send_error(KCONSUMERD_SPLICE_ENOMEM
);
328 send_error(KCONSUMERD_SPLICE_ESPIPE
);
339 * Consume data on a file descriptor and write it on a trace file
341 static int read_subbuffer(struct ltt_kconsumerd_fd
*kconsumerd_fd
)
346 int infd
= kconsumerd_fd
->consumerd_fd
;
348 DBG("In read_subbuffer (infd : %d)", infd
);
349 /* Get the next subbuffer */
350 err
= kernctl_get_next_subbuf(infd
);
353 perror("Reserving sub buffer failed (everything is normal, "
354 "it is due to concurrency)");
358 /* read the whole subbuffer */
359 err
= kernctl_get_padded_subbuf_size(infd
, &len
);
362 perror("Getting sub-buffer len failed.");
366 /* splice the subbuffer to the tracefile */
367 ret
= on_read_subbuffer(kconsumerd_fd
, len
);
370 * display the error but continue processing to try
371 * to release the subbuffer
373 ERR("Error splicing to tracefile");
376 err
= kernctl_put_next_subbuf(infd
);
379 if (errno
== EFAULT
) {
380 perror("Error in unreserving sub buffer\n");
381 } else if (errno
== EIO
) {
382 /* Should never happen with newer LTTng versions */
383 perror("Reader has been pushed by the writer, last sub-buffer corrupted.");
395 * Update a fd according to what we just received
397 static void change_fd_state(int sessiond_fd
,
398 enum kconsumerd_fd_state state
)
400 struct ltt_kconsumerd_fd
*iter
;
401 cds_list_for_each_entry(iter
, &kconsumerd_fd_list
.head
, list
) {
402 if (iter
->sessiond_fd
== sessiond_fd
) {
412 * Receives an array of file descriptors and the associated
413 * structures describing each fd (path name).
414 * Returns the size of received data
416 static int consumerd_recv_fd(int sfd
, int size
,
417 enum kconsumerd_command cmd_type
)
421 int ret
= 0, i
, tmp2
;
422 struct cmsghdr
*cmsg
;
424 char recv_fd
[CMSG_SPACE(sizeof(int))];
425 struct lttcomm_kconsumerd_msg lkm
;
427 /* the number of fds we are about to receive */
428 nb_fd
= size
/ sizeof(struct lttcomm_kconsumerd_msg
);
430 for (i
= 0; i
< nb_fd
; i
++) {
431 memset(&msg
, 0, sizeof(msg
));
433 /* Prepare to receive the structures */
434 iov
[0].iov_base
= &lkm
;
435 iov
[0].iov_len
= sizeof(lkm
);
439 msg
.msg_control
= recv_fd
;
440 msg
.msg_controllen
= sizeof(recv_fd
);
442 DBG("Waiting to receive fd");
443 if ((ret
= recvmsg(sfd
, &msg
, 0)) < 0) {
448 if (ret
!= (size
/ nb_fd
)) {
449 ERR("Received only %d, expected %d", ret
, size
);
450 send_error(KCONSUMERD_ERROR_RECV_FD
);
454 cmsg
= CMSG_FIRSTHDR(&msg
);
456 ERR("Invalid control message header");
458 send_error(KCONSUMERD_ERROR_RECV_FD
);
462 /* if we received fds */
463 if (cmsg
->cmsg_level
== SOL_SOCKET
&& cmsg
->cmsg_type
== SCM_RIGHTS
) {
466 DBG("add_fd %s (%d)", lkm
.path_name
, (CMSG_DATA(cmsg
)[0]));
467 ret
= add_fd(&lkm
, (CMSG_DATA(cmsg
)[0]));
469 send_error(KCONSUMERD_OUTFD_ERROR
);
474 change_fd_state(lkm
.fd
, lkm
.state
);
479 /* flag to tell the polling thread to update its fd array */
481 /* signal the poll thread */
482 tmp2
= write(poll_pipe
[1], "4", 1);
484 ERR("Didn't received any fd");
485 send_error(KCONSUMERD_ERROR_RECV_FD
);
492 DBG("consumerd_recv_fd thread exiting");
499 * This thread listens on the consumerd socket and
500 * receives the file descriptors from ltt-sessiond
502 static void *thread_receive_fds(void *data
)
504 int sock
, client_socket
, ret
;
505 struct lttcomm_kconsumerd_header tmp
;
507 DBG("Creating command socket %s", command_sock_path
);
508 unlink(command_sock_path
);
509 client_socket
= lttcomm_create_unix_sock(command_sock_path
);
510 if (client_socket
< 0) {
511 ERR("Cannot create command socket");
515 ret
= lttcomm_listen_unix_sock(client_socket
);
520 DBG("Sending ready command to ltt-sessiond");
521 ret
= send_error(KCONSUMERD_COMMAND_SOCK_READY
);
523 ERR("Error sending ready command to ltt-sessiond");
527 /* Blocking call, waiting for transmission */
528 sock
= lttcomm_accept_unix_sock(client_socket
);
534 /* We first get the number of fd we are about to receive */
535 ret
= lttcomm_recv_unix_sock(sock
, &tmp
,
536 sizeof(struct lttcomm_kconsumerd_header
));
538 ERR("Communication interrupted on command socket");
541 if (tmp
.cmd_type
== STOP
) {
542 DBG("Received STOP command");
545 /* we received a command to add or update fds */
546 ret
= consumerd_recv_fd(sock
, tmp
.payload_size
, tmp
.cmd_type
);
548 ERR("Receiving the FD, exiting");
554 DBG("thread_receive_fds exiting");
556 ret
= write(poll_pipe
[1], "4", 1);
558 perror("poll pipe write");
566 * Allocate the pollfd structure and the local view of the out fds
567 * to avoid doing a lookup in the linked list and concurrency issues
568 * when writing is needed.
569 * Returns the number of fds in the structures
571 static int update_poll_array(struct pollfd
**pollfd
,
572 struct ltt_kconsumerd_fd
**local_kconsumerd_fd
)
574 struct ltt_kconsumerd_fd
*iter
;
578 DBG("Updating poll fd array");
579 pthread_mutex_lock(&kconsumerd_lock_fds
);
581 cds_list_for_each_entry(iter
, &kconsumerd_fd_list
.head
, list
) {
582 DBG("Inside for each");
583 if (iter
->state
== ACTIVE_FD
) {
584 DBG("Active FD %d", iter
->consumerd_fd
);
585 (*pollfd
)[i
].fd
= iter
->consumerd_fd
;
586 (*pollfd
)[i
].events
= POLLIN
| POLLPRI
;
587 local_kconsumerd_fd
[i
] = iter
;
592 * insert the poll_pipe at the end of the array and don't increment i
593 * so nb_fd is the number of real FD
595 (*pollfd
)[i
].fd
= poll_pipe
[0];
596 (*pollfd
)[i
].events
= POLLIN
;
599 pthread_mutex_unlock(&kconsumerd_lock_fds
);
607 * This thread polls the fds in the ltt_fd_list to consume the data
608 * and write it to tracefile if necessary.
610 static void *thread_poll_fds(void *data
)
612 int num_rdy
, num_hup
, high_prio
, ret
, i
;
613 struct pollfd
*pollfd
= NULL
;
614 /* local view of the fds */
615 struct ltt_kconsumerd_fd
**local_kconsumerd_fd
= NULL
;
616 /* local view of fds_count */
621 ret
= pipe(thread_pipe
);
623 perror("Error creating pipe");
627 local_kconsumerd_fd
= malloc(sizeof(struct ltt_kconsumerd_fd
));
634 * the ltt_fd_list has been updated, we need to update our
635 * local array as well
637 if (update_fd_array
== 1) {
638 if (pollfd
!= NULL
) {
642 if (local_kconsumerd_fd
!= NULL
) {
643 free(local_kconsumerd_fd
);
644 local_kconsumerd_fd
= NULL
;
646 /* allocate for all fds + 1 for the poll_pipe */
647 pollfd
= malloc((fds_count
+ 1) * sizeof(struct pollfd
));
648 if (pollfd
== NULL
) {
649 perror("pollfd malloc");
652 /* allocate for all fds + 1 for the poll_pipe */
653 local_kconsumerd_fd
= malloc((fds_count
+ 1) * sizeof(struct ltt_kconsumerd_fd
));
654 if (local_kconsumerd_fd
== NULL
) {
655 perror("local_kconsumerd_fd malloc");
659 ret
= update_poll_array(&pollfd
, local_kconsumerd_fd
);
661 ERR("Error in allocating pollfd or local_outfds");
662 send_error(KCONSUMERD_POLL_ERROR
);
668 /* poll on the array of fds */
669 DBG("polling on %d fd", nb_fd
+ 1);
670 num_rdy
= poll(pollfd
, nb_fd
+ 1, -1);
671 DBG("poll num_rdy : %d", num_rdy
);
673 perror("Poll error");
674 send_error(KCONSUMERD_POLL_ERROR
);
678 /* No FDs and quit, cleanup the thread */
679 if (nb_fd
== 0 && quit
== 1) {
684 * if only the poll_pipe triggered poll to return just return to the
685 * beginning of the loop to update the array
687 if (num_rdy
== 1 && pollfd
[nb_fd
].revents
== POLLIN
) {
688 DBG("poll_pipe wake up");
689 tmp2
= read(poll_pipe
[0], &tmp
, 1);
693 /* Take care of high priority channels first. */
694 for (i
= 0; i
< nb_fd
; i
++) {
695 switch(pollfd
[i
].revents
) {
697 ERR("Error returned in polling fd %d.", pollfd
[i
].fd
);
698 del_fd(local_kconsumerd_fd
[i
]);
703 ERR("Polling fd %d tells it has hung up.", pollfd
[i
].fd
);
704 del_fd(local_kconsumerd_fd
[i
]);
709 ERR("Polling fd %d tells fd is not open.", pollfd
[i
].fd
);
710 del_fd(local_kconsumerd_fd
[i
]);
715 DBG("Urgent read on fd %d", pollfd
[i
].fd
);
717 ret
= read_subbuffer(local_kconsumerd_fd
[i
]);
718 /* it's ok to have an unavailable sub-buffer (FIXME : is it ?) */
726 /* If every buffer FD has hung up, we end the read loop here */
727 if (nb_fd
> 0 && num_hup
== nb_fd
) {
728 DBG("every buffer FD has hung up\n");
735 /* Take care of low priority channels. */
736 if (high_prio
== 0) {
737 for (i
= 0; i
< nb_fd
; i
++) {
738 if (pollfd
[i
].revents
== POLLIN
) {
739 DBG("Normal read on fd %d", pollfd
[i
].fd
);
740 ret
= read_subbuffer(local_kconsumerd_fd
[i
]);
741 /* it's ok to have an unavailable subbuffer (FIXME : is it ?) */
750 DBG("polling thread exiting");
751 if (pollfd
!= NULL
) {
755 if (local_kconsumerd_fd
!= NULL
) {
756 free(local_kconsumerd_fd
);
757 local_kconsumerd_fd
= NULL
;
764 * usage function on stderr
766 static void usage(void)
768 fprintf(stderr
, "Usage: %s OPTIONS\n\nOptions:\n", progname
);
769 fprintf(stderr
, " -h, --help "
770 "Display this usage.\n");
771 fprintf(stderr
, " -c, --kconsumerd-cmd-sock PATH "
772 "Specify path for the command socket\n");
773 fprintf(stderr
, " -e, --kconsumerd-err-sock PATH "
774 "Specify path for the error socket\n");
775 fprintf(stderr
, " -d, --daemonize "
776 "Start as a daemon.\n");
777 fprintf(stderr
, " -q, --quiet "
778 "No output at all.\n");
779 fprintf(stderr
, " -v, --verbose "
780 "Verbose mode. Activate DBG() macro.\n");
781 fprintf(stderr
, " -V, --version "
782 "Show version number.\n");
786 * daemon argument parsing
788 static void parse_args(int argc
, char **argv
)
792 static struct option long_options
[] = {
793 { "kconsumerd-cmd-sock", 1, 0, 'c' },
794 { "kconsumerd-err-sock", 1, 0, 'e' },
795 { "daemonize", 0, 0, 'd' },
796 { "help", 0, 0, 'h' },
797 { "quiet", 0, 0, 'q' },
798 { "verbose", 0, 0, 'v' },
799 { "version", 0, 0, 'V' },
804 int option_index
= 0;
805 c
= getopt_long(argc
, argv
, "dhqvV" "c:e:", long_options
, &option_index
);
812 fprintf(stderr
, "option %s", long_options
[option_index
].name
);
814 fprintf(stderr
, " with arg %s\n", optarg
);
818 snprintf(command_sock_path
, PATH_MAX
, "%s", optarg
);
821 snprintf(error_sock_path
, PATH_MAX
, "%s", optarg
);
836 fprintf(stdout
, "%s\n", VERSION
);
849 int main(int argc
, char **argv
)
855 /* Parse arguments */
857 parse_args(argc
, argv
);
868 if (strlen(command_sock_path
) == 0) {
869 snprintf(command_sock_path
, PATH_MAX
,
870 KCONSUMERD_CMD_SOCK_PATH
);
872 if (strlen(error_sock_path
) == 0) {
873 snprintf(error_sock_path
, PATH_MAX
,
874 KCONSUMERD_ERR_SOCK_PATH
);
877 if (set_signal_handler() < 0) {
881 /* create the pipe to wake to polling thread when needed */
882 ret
= pipe(poll_pipe
);
884 perror("Error creating poll pipe");
888 /* Connect to the socket created by ltt-sessiond to report errors */
889 DBG("Connecting to error socket %s", error_sock_path
);
890 error_socket
= lttcomm_connect_unix_sock(error_sock_path
);
891 /* not a fatal error, but all communication with ltt-sessiond will fail */
892 if (error_socket
< 0) {
893 WARN("Cannot connect to error socket, is ltt-sessiond started ?");
896 /* Create the thread to manage the receive of fd */
897 ret
= pthread_create(&threads
[0], NULL
, thread_receive_fds
, (void *) NULL
);
899 perror("pthread_create");
903 /* Create thread to manage the polling/writing of traces */
904 ret
= pthread_create(&threads
[1], NULL
, thread_poll_fds
, (void *) NULL
);
906 perror("pthread_create");
910 for (i
= 0; i
< 2; i
++) {
911 ret
= pthread_join(threads
[i
], &status
);
913 perror("pthread_join");
918 send_error(KCONSUMERD_EXIT_SUCCESS
);
923 send_error(KCONSUMERD_EXIT_FAILURE
);