libust/buffers.h

   1 /*
   2  * buffers.h
   3  *
   4  * Copyright (C) 2009 - Pierre-Marc Fournier (pierre-marc dot fournier at polymtl dot ca)
   5  * Copyright (C) 2008 - Mathieu Desnoyers (mathieu.desnoyers@polymtl.ca)
   6  *
   7  */
   8
   9 #ifndef _UST_BUFFERS_H
  10 #define _UST_BUFFERS_H
  11
  12 #include <kcompat/kref.h>
  13 #include <assert.h>
  14 #include "channels.h"
  15 #include "buffers.h"
  16
  17 /* Return the size of the minimum number of pages that can contain x. */
  18 #define FIX_SIZE(x) ((((x) - 1) & PAGE_MASK) + PAGE_SIZE)
  19
  20 /*
  21  * BUFFER_TRUNC zeroes the subbuffer offset and the subbuffer number parts of
  22  * the offset, which leaves only the buffer number.
  23  */
  24 #define BUFFER_TRUNC(offset, chan) \
  25         ((offset) & (~((chan)->alloc_size-1)))
  26 #define BUFFER_OFFSET(offset, chan) ((offset) & ((chan)->alloc_size - 1))
  27 #define SUBBUF_OFFSET(offset, chan) ((offset) & ((chan)->subbuf_size - 1))
  28 #define SUBBUF_ALIGN(offset, chan) \
  29         (((offset) + (chan)->subbuf_size) & (~((chan)->subbuf_size - 1)))
  30 #define SUBBUF_TRUNC(offset, chan) \
  31         ((offset) & (~((chan)->subbuf_size - 1)))
  32 #define SUBBUF_INDEX(offset, chan) \
  33         (BUFFER_OFFSET((offset), chan) >> (chan)->subbuf_size_order)
  34
  35 /*
  36  * Tracks changes to rchan/rchan_buf structs
  37  */
  38 #define UST_CHANNEL_VERSION             8
  39
  40 struct ust_buffer {
  41         /* First 32 bytes cache-hot cacheline */
  42         local_t offset;                 /* Current offset in the buffer */
  43         local_t *commit_count;          /* Commit count per sub-buffer */
  44         atomic_long_t consumed;         /*
  45                                          * Current offset in the buffer
  46                                          * standard atomic access (shared)
  47                                          */
  48         unsigned long last_tsc;         /*
  49                                          * Last timestamp written in the buffer.
  50                                          */
  51         /* End of first 32 bytes cacheline */
  52         atomic_long_t active_readers;   /*
  53                                          * Active readers count
  54                                          * standard atomic access (shared)
  55                                          */
  56         local_t events_lost;
  57         local_t corrupted_subbuffers;
  58         /* one byte is written to this pipe when data is available, in order
  59            to wake the consumer */
  60         /* portability: Single byte writes must be as quick as possible. The kernel-side
  61            buffer must be large enough so the writer doesn't block. From the pipe(7)
  62            man page: Since linux 2.6.11, the pipe capacity is 65536 bytes. */
  63         int data_ready_fd_write;
  64         /* the reading end of the pipe */
  65         int data_ready_fd_read;
  66
  67         struct ust_channel *chan;
  68         struct kref kref;
  69         void *buf_data;
  70         size_t buf_size;
  71         int shmid;
  72         unsigned int cpu;
  73
  74         /* commit count per subbuffer; must be at end of struct */
  75         local_t commit_seq[0] ____cacheline_aligned;
  76 } ____cacheline_aligned;
  77
  78 extern void _ust_buffers_write(struct ust_buffer *buf, size_t offset,
  79         const void *src, size_t len, ssize_t cpy);
  80
  81 /*
  82  * Return the address where a given offset is located.
  83  * Should be used to get the current subbuffer header pointer. Given we know
  84  * it's never on a page boundary, it's safe to write directly to this address,
  85  * as long as the write is never bigger than a page size.
  86  */
  87 extern void *ust_buffers_offset_address(struct ust_buffer *buf,
  88         size_t offset);
  89
  90 /* FIXME: lttng has a version for systems with inefficient unaligned access */
  91 static __inline__ void ust_buffers_do_copy(void *dest, const void *src, size_t len)
  92 {
  93         union {
  94                 const void *src;
  95                 const u8 *src8;
  96                 const u16 *src16;
  97                 const u32 *src32;
  98                 const u64 *src64;
  99         } u = { .src = src };
 100
 101         switch (len) {
 102         case 0: break;
 103         case 1: *(u8 *)dest = *u.src8;
 104                 break;
 105         case 2: *(u16 *)dest = *u.src16;
 106                 break;
 107         case 4: *(u32 *)dest = *u.src32;
 108                 break;
 109         case 8: *(u64 *)dest = *u.src64;
 110                 break;
 111         default:
 112                 memcpy(dest, src, len);
 113         }
 114 }
 115
 116 /* FIXME: there is both a static inline and a '_' non static inline version ?? */
 117 static __inline__ int ust_buffers_write(struct ust_buffer *buf, size_t offset,
 118         const void *src, size_t len)
 119 {
 120         size_t cpy;
 121         size_t buf_offset = BUFFER_OFFSET(offset, buf->chan);
 122
 123         assert(buf_offset < buf->chan->subbuf_size*buf->chan->subbuf_cnt);
 124
 125         cpy = min_t(size_t, len, buf->buf_size - buf_offset);
 126         ust_buffers_do_copy(buf->buf_data + buf_offset, src, cpy);
 127
 128         if (unlikely(len != cpy))
 129                 _ust_buffers_write(buf, buf_offset, src, len, cpy);
 130         return len;
 131 }
 132
 133 int ust_buffers_channel_open(struct ust_channel *chan, size_t subbuf_size, size_t n_subbufs);
 134 extern void ust_buffers_channel_close(struct ust_channel *chan);
 135
 136 extern int ust_buffers_do_get_subbuf(struct ust_buffer *buf, long *pconsumed_old);
 137
 138 extern int ust_buffers_do_put_subbuf(struct ust_buffer *buf, u32 uconsumed_old);
 139
 140 extern void init_ustrelay_transport(void);
 141
 142 /*static*/ /* inline */ notrace void ltt_commit_slot(
 143                 struct ust_channel *channel,
 144                 void **transport_data, long buf_offset,
 145                 size_t data_size, size_t slot_size);
 146
 147 #endif /* _UST_BUFFERS_H */