libust/buffers.h

   1 /*
   2  * buffers.h
   3  *
   4  * Copyright (C) 2009 - Pierre-Marc Fournier (pierre-marc dot fournier at polymtl dot ca)
   5  * Copyright (C) 2008 - Mathieu Desnoyers (mathieu.desnoyers@polymtl.ca)
   6  *
   7  */
   8
   9 #ifndef _UST_BUFFERS_H
  10 #define _UST_BUFFERS_H
  11
  12 #include <kcompat/kref.h>
  13 #include <assert.h>
  14 #include "channels.h"
  15 #include "buffers.h"
  16
  17 /* Return the size of the minimum number of pages that can contain x. */
  18 #define FIX_SIZE(x) ((((x) - 1) & PAGE_MASK) + PAGE_SIZE)
  19
  20 /*
  21  * BUFFER_TRUNC zeroes the subbuffer offset and the subbuffer number parts of
  22  * the offset, which leaves only the buffer number.
  23  */
  24 #define BUFFER_TRUNC(offset, chan) \
  25         ((offset) & (~((chan)->alloc_size-1)))
  26 #define BUFFER_OFFSET(offset, chan) ((offset) & ((chan)->alloc_size - 1))
  27 #define SUBBUF_OFFSET(offset, chan) ((offset) & ((chan)->subbuf_size - 1))
  28 #define SUBBUF_ALIGN(offset, chan) \
  29         (((offset) + (chan)->subbuf_size) & (~((chan)->subbuf_size - 1)))
  30 #define SUBBUF_TRUNC(offset, chan) \
  31         ((offset) & (~((chan)->subbuf_size - 1)))
  32 #define SUBBUF_INDEX(offset, chan) \
  33         (BUFFER_OFFSET((offset), chan) >> (chan)->subbuf_size_order)
  34
  35 /*
  36  * Tracks changes to rchan/rchan_buf structs
  37  */
  38 #define UST_CHANNEL_VERSION             8
  39
  40 struct ust_buffer {
  41         /* First 32 bytes cache-hot cacheline */
  42         local_t offset;                 /* Current offset in the buffer */
  43         local_t *commit_count;          /* Commit count per sub-buffer */
  44         atomic_long_t consumed;         /*
  45                                          * Current offset in the buffer
  46                                          * standard atomic access (shared)
  47                                          */
  48         unsigned long last_tsc;         /*
  49                                          * Last timestamp written in the buffer.
  50                                          */
  51         /* End of first 32 bytes cacheline */
  52         atomic_long_t active_readers;   /*
  53                                          * Active readers count
  54                                          * standard atomic access (shared)
  55                                          */
  56         local_t events_lost;
  57         local_t corrupted_subbuffers;
  58         /* one byte is written to this pipe when data is available, in order
  59            to wake the consumer */
  60         /* portability: Single byte writes must be as quick as possible. The kernel-side
  61            buffer must be large enough so the writer doesn't block. From the pipe(7)
  62            man page: Since linux 2.6.11, the pipe capacity is 65536 bytes. */
  63         int data_ready_fd_write;
  64         /* the reading end of the pipe */
  65         int data_ready_fd_read;
  66
  67         struct ust_channel *chan;
  68         struct kref kref;
  69         void *buf_data;
  70         size_t buf_size;
  71         int shmid;
  72
  73         /* commit count per subbuffer; must be at end of struct */
  74         local_t commit_seq[0] ____cacheline_aligned;
  75 } ____cacheline_aligned;
  76
  77 extern void _ust_buffers_write(struct ust_buffer *buf, size_t offset,
  78         const void *src, size_t len, ssize_t cpy);
  79
  80 /*
  81  * Return the address where a given offset is located.
  82  * Should be used to get the current subbuffer header pointer. Given we know
  83  * it's never on a page boundary, it's safe to write directly to this address,
  84  * as long as the write is never bigger than a page size.
  85  */
  86 extern void *ust_buffers_offset_address(struct ust_buffer *buf,
  87         size_t offset);
  88
  89 /* FIXME: lttng has a version for systems with inefficient unaligned access */
  90 static inline void ust_buffers_do_copy(void *dest, const void *src, size_t len)
  91 {
  92         union {
  93                 const void *src;
  94                 const u8 *src8;
  95                 const u16 *src16;
  96                 const u32 *src32;
  97                 const u64 *src64;
  98         } u = { .src = src };
  99
 100         switch (len) {
 101         case 0: break;
 102         case 1: *(u8 *)dest = *u.src8;
 103                 break;
 104         case 2: *(u16 *)dest = *u.src16;
 105                 break;
 106         case 4: *(u32 *)dest = *u.src32;
 107                 break;
 108         case 8: *(u64 *)dest = *u.src64;
 109                 break;
 110         default:
 111                 memcpy(dest, src, len);
 112         }
 113 }
 114
 115 /* FIXME: there is both a static inline and a '_' non static inline version ?? */
 116 static inline int ust_buffers_write(struct ust_buffer *buf, size_t offset,
 117         const void *src, size_t len)
 118 {
 119         size_t cpy;
 120         size_t buf_offset = BUFFER_OFFSET(offset, buf->chan);
 121
 122         assert(buf_offset < buf->chan->subbuf_size*buf->chan->subbuf_cnt);
 123
 124         cpy = min_t(size_t, len, buf->buf_size - buf_offset);
 125         ust_buffers_do_copy(buf->buf_data + buf_offset, src, cpy);
 126
 127         if (unlikely(len != cpy))
 128                 _ust_buffers_write(buf, buf_offset, src, len, cpy);
 129         return len;
 130 }
 131
 132 int ust_buffers_channel_open(struct ust_channel *chan, size_t subbuf_size, size_t n_subbufs);
 133 extern void ust_buffers_channel_close(struct ust_channel *chan);
 134
 135 extern int ust_buffers_do_get_subbuf(struct ust_buffer *buf, long *pconsumed_old);
 136
 137 extern int ust_buffers_do_put_subbuf(struct ust_buffer *buf, u32 uconsumed_old);
 138
 139 extern void init_ustrelay_transport(void);
 140
 141 /*static*/ /* inline */ notrace void ltt_commit_slot(
 142                 struct ust_channel *channel,
 143                 void **transport_data, long buf_offset,
 144                 size_t data_size, size_t slot_size);
 145
 146 #endif /* _UST_BUFFERS_H */