ltt-relay.h

   1  /*
   2  * include/linux/ltt-relay.h
   3  *
   4  * Copyright (C) 2008,2009 - Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
   5  *
   6  * Dual LGPL v2.1/GPL v2 license.
   7  *
   8  * Credits to Steven Rostedt for proposing to use an extra-subbuffer owned by
   9  * the reader in flight recorder mode.
  10  */
  11
  12 #ifndef _LINUX_LTT_RELAY_H
  13 #define _LINUX_LTT_RELAY_H
  14
  15 #include <linux/types.h>
  16 #include <linux/sched.h>
  17 #include <linux/timer.h>
  18 #include <linux/wait.h>
  19 #include <linux/fs.h>
  20 #include <linux/poll.h>
  21 #include <linux/kref.h>
  22 #include <linux/mm.h>
  23 #include <linux/ltt-channels.h>
  24
  25 #include "ltt-tracer-core.h"
  26
  27 /* Use lowest pointer bit to show the sub-buffer has no reference. */
  28 #define RCHAN_NOREF_FLAG        0x1UL
  29
  30 #define RCHAN_SB_IS_NOREF(x)    ((unsigned long)(x) & RCHAN_NOREF_FLAG)
  31 #define RCHAN_SB_SET_NOREF(x)   \
  32         (x = (struct chanbuf_page *)((unsigned long)(x) | RCHAN_NOREF_FLAG))
  33 #define RCHAN_SB_CLEAR_NOREF(x) \
  34         (x = (struct chanbuf_page *)((unsigned long)(x) & ~RCHAN_NOREF_FLAG))
  35
  36 struct ltt_trace;
  37
  38 struct chanbuf_page {
  39         void *virt;                     /* page virtual address (cached) */
  40         struct page *page;              /* pointer to page structure */
  41 };
  42
  43 struct chanbuf_sb {
  44         struct chanbuf_page *pages;     /* Pointer to rchan pages for subbuf */
  45 };
  46
  47 struct ltt_chanbuf_alloc {
  48         struct chanbuf_sb *buf_wsb;     /* Array of rchan_sb for writer */
  49         struct chanbuf_sb buf_rsb;      /* chanbuf_sb for reader */
  50         void **_virt;                   /* Array of pointers to page addr */
  51         struct page **_pages;           /* Array of pointers to pages */
  52         struct dentry *dentry;          /* Associated file dentry */
  53         unsigned int nr_pages;          /* Number pages in buffer */
  54
  55         struct ltt_chan_alloc *chan;    /* Associated channel */
  56         unsigned int cpu;               /* This buffer's cpu */
  57         unsigned int allocated:1;       /* Bool: is buffer allocated ? */
  58 };
  59
  60 int ltt_chanbuf_alloc_create(struct ltt_chanbuf_alloc *buf,
  61                              struct ltt_chan_alloc *chan, int cpu);
  62 void ltt_chanbuf_alloc_free(struct ltt_chanbuf_alloc *buf);
  63 int ltt_chan_alloc_init(struct ltt_chan_alloc *chan, struct ltt_trace *trace,
  64                         const char *base_filename,
  65                         struct dentry *parent, size_t sb_size,
  66                         size_t n_sb, int extra_reader_sb, int overwrite);
  67 void ltt_chan_alloc_free(struct ltt_chan_alloc *chan);
  68 void ltt_chan_alloc_remove_files(struct ltt_chan_alloc *chan);
  69 int ltt_chanbuf_create_file(const char *filename, struct dentry *parent,
  70                             int mode, struct ltt_chanbuf *buf);
  71 int ltt_chanbuf_remove_file(struct ltt_chanbuf *buf);
  72
  73 void ltt_chan_for_each_channel(void (*cb) (struct ltt_chanbuf *buf), int cpu);
  74
  75 extern void _ltt_relay_write(struct ltt_chanbuf_alloc *bufa,
  76                              size_t offset, const void *src, size_t len,
  77                              ssize_t pagecpy);
  78
  79 extern void _ltt_relay_strncpy(struct ltt_chanbuf_alloc *bufa,
  80                                size_t offset, const void *src, size_t len,
  81                                ssize_t pagecpy);
  82
  83 extern void _ltt_relay_strncpy_fixup(struct ltt_chanbuf_alloc *bufa,
  84                                      size_t offset, size_t len, size_t copied,
  85                                      int terminated);
  86
  87 extern int ltt_relay_read(struct ltt_chanbuf_alloc *bufa,
  88                           size_t offset, void *dest, size_t len);
  89
  90 extern int ltt_relay_read_cstr(struct ltt_chanbuf_alloc *bufa,
  91                                size_t offset, void *dest, size_t len);
  92
  93 extern struct page *ltt_relay_read_get_page(struct ltt_chanbuf_alloc *bufa,
  94                                             size_t offset);
  95
  96 /*
  97  * Return the address where a given offset is located.
  98  * Should be used to get the current subbuffer header pointer. Given we know
  99  * it's never on a page boundary, it's safe to write directly to this address,
 100  * as long as the write is never bigger than a page size.
 101  */
 102 extern void *ltt_relay_offset_address(struct ltt_chanbuf_alloc *bufa,
 103                                       size_t offset);
 104 extern void *ltt_relay_read_offset_address(struct ltt_chanbuf_alloc *bufa,
 105                                            size_t offset);
 106
 107 #ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
 108 static __inline__
 109 void ltt_relay_do_copy(void *dest, const void *src, size_t len)
 110 {
 111         switch (len) {
 112         case 0:
 113                 break;
 114         case 1:
 115                 *(u8 *)dest = *(const u8 *)src;
 116                 break;
 117         case 2:
 118                 *(u16 *)dest = *(const u16 *)src;
 119                 break;
 120         case 4:
 121                 *(u32 *)dest = *(const u32 *)src;
 122                 break;
 123         case 8:
 124                 *(u64 *)dest = *(const u64 *)src;
 125                 break;
 126         default:
 127                 /*
 128                  * What we really want here is an __inline__ memcpy, but we don't
 129                  * have constants, so gcc generally uses a function call.
 130                  */
 131                 for (; len > 0; len--)
 132                         *(u8 *)dest++ = *(const u8 *)src++;
 133         }
 134 }
 135 #else
 136 /*
 137  * Returns whether the dest and src addresses are aligned on
 138  * min(sizeof(void *), len). Call this with statically known len for efficiency.
 139  */
 140 static __inline__
 141 int addr_aligned(const void *dest, const void *src, size_t len)
 142 {
 143         if (ltt_align((size_t)dest, len))
 144                 return 0;
 145         if (ltt_align((size_t)src, len))
 146                 return 0;
 147         return 1;
 148 }
 149
 150 static __inline__
 151 void ltt_relay_do_copy(void *dest, const void *src, size_t len)
 152 {
 153         switch (len) {
 154         case 0:
 155                 break;
 156         case 1:
 157                 *(u8 *)dest = *(const u8 *)src;
 158                 break;
 159         case 2:
 160                 if (unlikely(!addr_aligned(dest, src, 2)))
 161                         goto memcpy_fallback;
 162                 *(u16 *)dest = *(const u16 *)src;
 163                 break;
 164         case 4:
 165                 if (unlikely(!addr_aligned(dest, src, 4)))
 166                         goto memcpy_fallback;
 167                 *(u32 *)dest = *(const u32 *)src;
 168                 break;
 169         case 8:
 170                 if (unlikely(!addr_aligned(dest, src, 8)))
 171                         goto memcpy_fallback;
 172                 *(u64 *)dest = *(const u64 *)src;
 173                 break;
 174         default:
 175                 goto memcpy_fallback;
 176         }
 177         return;
 178
 179 memcpy_fallback:
 180         /*
 181          * What we really want here is an inline memcpy, but we don't
 182          * have constants, so gcc generally uses a function call.
 183          */
 184         for (; len > 0; len--)
 185                 *(u8 *)dest++ = *(const u8 *)src++;
 186 }
 187 #endif
 188
 189 /*
 190  * ltt_relay_do_memset - write character into dest.
 191  * @dest: destination
 192  * @src: source character
 193  * @len: length to write
 194  */
 195 static __inline__
 196 void ltt_relay_do_memset(void *dest, char src, size_t len)
 197 {
 198         /*
 199          * What we really want here is an __inline__ memset, but we
 200          * don't have constants, so gcc generally uses a function call.
 201          */
 202         for (; len > 0; len--)
 203                 *(u8 *)dest++ = src;
 204 }
 205
 206
 207 /*
 208  * ltt_relay_do_strncpy - copy a string up to a certain number of bytes
 209  * @dest: destination
 210  * @src: source
 211  * @len: max. length to copy
 212  * @terminated: output string ends with \0 (output)
 213  *
 214  * returns the number of bytes copied. Does not finalize with \0 if len is
 215  * reached.
 216  */
 217 static __inline__
 218 size_t ltt_relay_do_strncpy(void *dest, const void *src, size_t len,
 219                             int *terminated)
 220 {
 221         size_t orig_len = len;
 222
 223         *terminated = 0;
 224         /*
 225          * What we really want here is an __inline__ strncpy, but we
 226          * don't have constants, so gcc generally uses a function call.
 227          */
 228         for (; len > 0; len--) {
 229                 *(u8 *)dest = ACCESS_ONCE(*(const u8 *)src);
 230                 /* Check with dest, because src may be modified concurrently */
 231                 if (*(const u8 *)dest == '\0') {
 232                         len--;
 233                         *terminated = 1;
 234                         break;
 235                 }
 236                 dest++;
 237                 src++;
 238         }
 239         return orig_len - len;
 240 }
 241
 242 static __inline__
 243 int ltt_relay_write(struct ltt_chanbuf_alloc *bufa,
 244                     struct ltt_chan_alloc *chana, size_t offset,
 245                     const void *src, size_t len)
 246 {
 247         size_t sbidx, index;
 248         ssize_t pagecpy;
 249         struct chanbuf_page *rpages;
 250
 251         offset &= chana->buf_size - 1;
 252         sbidx = offset >> chana->sb_size_order;
 253         index = (offset & (chana->sb_size - 1)) >> PAGE_SHIFT;
 254         pagecpy = min_t(size_t, len, (- offset) & ~PAGE_MASK);
 255         rpages = bufa->buf_wsb[sbidx].pages;
 256         WARN_ON_ONCE(RCHAN_SB_IS_NOREF(rpages));
 257         ltt_relay_do_copy(rpages[index].virt + (offset & ~PAGE_MASK),
 258                           src, pagecpy);
 259
 260         if (unlikely(len != pagecpy))
 261                 _ltt_relay_write(bufa, offset, src, len, pagecpy);
 262         return len;
 263 }
 264
 265 static __inline__
 266 int ltt_relay_strncpy(struct ltt_chanbuf_alloc *bufa,
 267                       struct ltt_chan_alloc *chana, size_t offset,
 268                       const void *src, size_t len)
 269 {
 270         size_t sbidx, index;
 271         ssize_t pagecpy, copied;
 272         struct chanbuf_page *rpages;
 273         int terminated;
 274
 275         offset &= chana->buf_size - 1;
 276         sbidx = offset >> chana->sb_size_order;
 277         index = (offset & (chana->sb_size - 1)) >> PAGE_SHIFT;
 278         pagecpy = min_t(size_t, len, (- offset) & ~PAGE_MASK);
 279         rpages = bufa->buf_wsb[sbidx].pages;
 280         WARN_ON_ONCE(RCHAN_SB_IS_NOREF(rpages));
 281         copied = ltt_relay_do_strncpy(rpages[index].virt
 282                                       + (offset & ~PAGE_MASK),
 283                                       src, pagecpy, &terminated);
 284         if (unlikely(copied < pagecpy || ((len == pagecpy) && !terminated)))
 285                 _ltt_relay_strncpy_fixup(bufa, offset, len, copied,
 286                                          terminated);
 287         else {
 288                 if (unlikely(len != pagecpy))
 289                         _ltt_relay_strncpy(bufa, offset, src, len, pagecpy);
 290         }
 291         return len;
 292 }
 293
 294 /**
 295  * ltt_clear_noref_flag - Clear the noref subbuffer flag, for writer.
 296  */
 297 static __inline__
 298 void ltt_clear_noref_flag(struct ltt_chanbuf_alloc *bufa, long idx)
 299 {
 300         struct chanbuf_page *sb_pages, *new_sb_pages;
 301
 302         sb_pages = bufa->buf_wsb[idx].pages;
 303         for (;;) {
 304                 if (!RCHAN_SB_IS_NOREF(sb_pages))
 305                         return; /* Already writing to this buffer */
 306                 new_sb_pages = sb_pages;
 307                 RCHAN_SB_CLEAR_NOREF(new_sb_pages);
 308                 new_sb_pages = cmpxchg(&bufa->buf_wsb[idx].pages,
 309                         sb_pages, new_sb_pages);
 310                 if (likely(new_sb_pages == sb_pages))
 311                         break;
 312                 sb_pages = new_sb_pages;
 313         }
 314 }
 315
 316 /**
 317  * ltt_set_noref_flag - Set the noref subbuffer flag, for writer.
 318  */
 319 static __inline__
 320 void ltt_set_noref_flag(struct ltt_chanbuf_alloc *bufa, long idx)
 321 {
 322         struct chanbuf_page *sb_pages, *new_sb_pages;
 323
 324         sb_pages = bufa->buf_wsb[idx].pages;
 325         for (;;) {
 326                 if (RCHAN_SB_IS_NOREF(sb_pages))
 327                         return; /* Already set */
 328                 new_sb_pages = sb_pages;
 329                 RCHAN_SB_SET_NOREF(new_sb_pages);
 330                 new_sb_pages = cmpxchg(&bufa->buf_wsb[idx].pages,
 331                         sb_pages, new_sb_pages);
 332                 if (likely(new_sb_pages == sb_pages))
 333                         break;
 334                 sb_pages = new_sb_pages;
 335         }
 336 }
 337
 338 /**
 339  * update_read_sb_index - Read-side subbuffer index update.
 340  */
 341 static __inline__
 342 int update_read_sb_index(struct ltt_chanbuf_alloc *bufa,
 343                          struct ltt_chan_alloc *chana,
 344                          long consumed_idx)
 345 {
 346         struct chanbuf_page *old_wpage, *new_wpage;
 347
 348         if (unlikely(chana->extra_reader_sb)) {
 349                 /*
 350                  * Exchange the target writer subbuffer with our own unused
 351                  * subbuffer.
 352                  */
 353                 old_wpage = bufa->buf_wsb[consumed_idx].pages;
 354                 if (unlikely(!RCHAN_SB_IS_NOREF(old_wpage)))
 355                         return -EAGAIN;
 356                 WARN_ON_ONCE(!RCHAN_SB_IS_NOREF(bufa->buf_rsb.pages));
 357                 new_wpage = cmpxchg(&bufa->buf_wsb[consumed_idx].pages,
 358                                 old_wpage,
 359                                 bufa->buf_rsb.pages);
 360                 if (unlikely(old_wpage != new_wpage))
 361                         return -EAGAIN;
 362                 bufa->buf_rsb.pages = new_wpage;
 363                 RCHAN_SB_CLEAR_NOREF(bufa->buf_rsb.pages);
 364         } else {
 365                 /* No page exchange, use the writer page directly */
 366                 bufa->buf_rsb.pages = bufa->buf_wsb[consumed_idx].pages;
 367                 RCHAN_SB_CLEAR_NOREF(bufa->buf_rsb.pages);
 368         }
 369         return 0;
 370 }
 371
 372 ssize_t ltt_relay_file_splice_read(struct file *in, loff_t *ppos,
 373                                    struct pipe_inode_info *pipe, size_t len,
 374                                    unsigned int flags);
 375 loff_t ltt_relay_no_llseek(struct file *file, loff_t offset, int origin);
 376
 377 extern int ltt_ascii_init(void);
 378 extern void ltt_ascii_exit(void);
 379
 380 #endif /* _LINUX_LTT_RELAY_H */