X-Git-Url: http://git.lttng.org/?a=blobdiff_plain;f=libringbuffer%2Fbackend_internal.h;h=e3192be7f224e8dadf2a424b5e243ee8a0c182a2;hb=4318ae1be57eb7983ab4857a7a8eeb4a030a8216;hp=c5f3362e53ed03c21267e635e5242a6c7299ccec;hpb=14641debd03ba299bd06040cb62e0dbdef7fac81;p=lttng-ust.git

diff --git a/libringbuffer/backend_internal.h b/libringbuffer/backend_internal.h
index c5f3362e..e3192be7 100644
--- a/libringbuffer/backend_internal.h
+++ b/libringbuffer/backend_internal.h
@@ -14,32 +14,37 @@
 #include <unistd.h>
 #include <urcu/compiler.h>
 
-#include "config.h"
+#include <lttng/ringbuffer-config.h>
 #include "backend_types.h"
 #include "frontend_types.h"
+#include "shm.h"
 
 /* Ring buffer backend API presented to the frontend */
 
 /* Ring buffer and channel backend create/free */
 
-int lib_ring_buffer_backend_create(struct lib_ring_buffer_backend *bufb,
-				   struct channel_backend *chan, int cpu);
+int lib_ring_buffer_backend_create(struct lttng_ust_lib_ring_buffer_backend *bufb,
+				   struct channel_backend *chan, int cpu,
+				   struct lttng_ust_shm_handle *handle,
+				   struct shm_object *shmobj);
 void channel_backend_unregister_notifiers(struct channel_backend *chanb);
-void lib_ring_buffer_backend_free(struct lib_ring_buffer_backend *bufb);
+void lib_ring_buffer_backend_free(struct lttng_ust_lib_ring_buffer_backend *bufb);
 int channel_backend_init(struct channel_backend *chanb,
 			 const char *name,
-			 const struct lib_ring_buffer_config *config,
+			 const struct lttng_ust_lib_ring_buffer_config *config,
 			 void *priv, size_t subbuf_size,
-			 size_t num_subbuf);
-void channel_backend_free(struct channel_backend *chanb);
+			 size_t num_subbuf, struct lttng_ust_shm_handle *handle);
+void channel_backend_free(struct channel_backend *chanb,
+			  struct lttng_ust_shm_handle *handle);
 
-void lib_ring_buffer_backend_reset(struct lib_ring_buffer_backend *bufb);
+void lib_ring_buffer_backend_reset(struct lttng_ust_lib_ring_buffer_backend *bufb,
+				   struct lttng_ust_shm_handle *handle);
 void channel_backend_reset(struct channel_backend *chanb);
 
 int lib_ring_buffer_backend_init(void);
 void lib_ring_buffer_backend_exit(void);
 
-extern void _lib_ring_buffer_write(struct lib_ring_buffer_backend *bufb,
+extern void _lib_ring_buffer_write(struct lttng_ust_lib_ring_buffer_backend *bufb,
 				   size_t offset, const void *src, size_t len,
 				   ssize_t pagecpy);
 
@@ -81,7 +86,7 @@ extern void _lib_ring_buffer_write(struct lib_ring_buffer_backend *bufb,
  * mode).
  */
 static inline
-unsigned long subbuffer_id(const struct lib_ring_buffer_config *config,
+unsigned long subbuffer_id(const struct lttng_ust_lib_ring_buffer_config *config,
 			   unsigned long offset, unsigned long noref,
 			   unsigned long index)
 {
@@ -98,14 +103,14 @@ unsigned long subbuffer_id(const struct lib_ring_buffer_config *config,
  * bits are identical, else 0.
  */
 static inline
-int subbuffer_id_compare_offset(const struct lib_ring_buffer_config *config,
+int subbuffer_id_compare_offset(const struct lttng_ust_lib_ring_buffer_config *config,
 				unsigned long id, unsigned long offset)
 {
 	return (id & SB_ID_OFFSET_MASK) == (offset << SB_ID_OFFSET_SHIFT);
 }
 
 static inline
-unsigned long subbuffer_id_get_index(const struct lib_ring_buffer_config *config,
+unsigned long subbuffer_id_get_index(const struct lttng_ust_lib_ring_buffer_config *config,
 				     unsigned long id)
 {
 	if (config->mode == RING_BUFFER_OVERWRITE)
@@ -115,7 +120,7 @@ unsigned long subbuffer_id_get_index(const struct lib_ring_buffer_config *config
 }
 
 static inline
-unsigned long subbuffer_id_is_noref(const struct lib_ring_buffer_config *config,
+unsigned long subbuffer_id_is_noref(const struct lttng_ust_lib_ring_buffer_config *config,
 				    unsigned long id)
 {
 	if (config->mode == RING_BUFFER_OVERWRITE)
@@ -129,7 +134,7 @@ unsigned long subbuffer_id_is_noref(const struct lib_ring_buffer_config *config,
  * needed.
  */
 static inline
-void subbuffer_id_set_noref(const struct lib_ring_buffer_config *config,
+void subbuffer_id_set_noref(const struct lttng_ust_lib_ring_buffer_config *config,
 			    unsigned long *id)
 {
 	if (config->mode == RING_BUFFER_OVERWRITE)
@@ -137,7 +142,7 @@ void subbuffer_id_set_noref(const struct lib_ring_buffer_config *config,
 }
 
 static inline
-void subbuffer_id_set_noref_offset(const struct lib_ring_buffer_config *config,
+void subbuffer_id_set_noref_offset(const struct lttng_ust_lib_ring_buffer_config *config,
 				   unsigned long *id, unsigned long offset)
 {
 	unsigned long tmp;
@@ -154,7 +159,7 @@ void subbuffer_id_set_noref_offset(const struct lib_ring_buffer_config *config,
 
 /* No volatile access, since already used locally */
 static inline
-void subbuffer_id_clear_noref(const struct lib_ring_buffer_config *config,
+void subbuffer_id_clear_noref(const struct lttng_ust_lib_ring_buffer_config *config,
 			      unsigned long *id)
 {
 	if (config->mode == RING_BUFFER_OVERWRITE)
@@ -169,7 +174,7 @@ void subbuffer_id_clear_noref(const struct lib_ring_buffer_config *config,
  * -EPERM on failure.
  */
 static inline
-int subbuffer_id_check_index(const struct lib_ring_buffer_config *config,
+int subbuffer_id_check_index(const struct lttng_ust_lib_ring_buffer_config *config,
 			     unsigned long num_subbuf)
 {
 	if (config->mode == RING_BUFFER_OVERWRITE)
@@ -179,14 +184,14 @@ int subbuffer_id_check_index(const struct lib_ring_buffer_config *config,
 }
 
 static inline
-void subbuffer_count_record(const struct lib_ring_buffer_config *config,
-			    struct lib_ring_buffer_backend *bufb,
-			    unsigned long idx)
+void subbuffer_count_record(const struct lttng_ust_lib_ring_buffer_config *config,
+			    struct lttng_ust_lib_ring_buffer_backend *bufb,
+			    unsigned long idx, struct lttng_ust_shm_handle *handle)
 {
 	unsigned long sb_bindex;
 
-	sb_bindex = subbuffer_id_get_index(config, bufb->buf_wsb[idx].id);
-	v_inc(config, &bufb->array[sb_bindex]->records_commit);
+	sb_bindex = subbuffer_id_get_index(config, shmp_index(handle, bufb->buf_wsb, idx)->id);
+	v_inc(config, &shmp(handle, shmp_index(handle, bufb->array, sb_bindex)->shmp)->records_commit);
 }
 
 /*
@@ -194,29 +199,31 @@ void subbuffer_count_record(const struct lib_ring_buffer_config *config,
  * perform the decrement atomically.
  */
 static inline
-void subbuffer_consume_record(const struct lib_ring_buffer_config *config,
-			      struct lib_ring_buffer_backend *bufb)
+void subbuffer_consume_record(const struct lttng_ust_lib_ring_buffer_config *config,
+			      struct lttng_ust_lib_ring_buffer_backend *bufb,
+			      struct lttng_ust_shm_handle *handle)
 {
 	unsigned long sb_bindex;
 
 	sb_bindex = subbuffer_id_get_index(config, bufb->buf_rsb.id);
-	CHAN_WARN_ON(bufb->chan,
-		     !v_read(config, &bufb->array[sb_bindex]->records_unread));
+	CHAN_WARN_ON(shmp(handle, bufb->chan),
+		     !v_read(config, &shmp(handle, shmp_index(handle, bufb->array, sb_bindex)->shmp)->records_unread));
 	/* Non-atomic decrement protected by exclusive subbuffer access */
-	_v_dec(config, &bufb->array[sb_bindex]->records_unread);
+	_v_dec(config, &shmp(handle, shmp_index(handle, bufb->array, sb_bindex)->shmp)->records_unread);
 	v_inc(config, &bufb->records_read);
 }
 
 static inline
 unsigned long subbuffer_get_records_count(
-				const struct lib_ring_buffer_config *config,
-				struct lib_ring_buffer_backend *bufb,
-				unsigned long idx)
+				const struct lttng_ust_lib_ring_buffer_config *config,
+				struct lttng_ust_lib_ring_buffer_backend *bufb,
+				unsigned long idx,
+				struct lttng_ust_shm_handle *handle)
 {
 	unsigned long sb_bindex;
 
-	sb_bindex = subbuffer_id_get_index(config, bufb->buf_wsb[idx].id);
-	return v_read(config, &bufb->array[sb_bindex]->records_commit);
+	sb_bindex = subbuffer_id_get_index(config, shmp_index(handle, bufb->buf_wsb, idx)->id);
+	return v_read(config, &shmp(handle, shmp_index(handle, bufb->array, sb_bindex)->shmp)->records_commit);
 }
 
 /*
@@ -227,62 +234,66 @@ unsigned long subbuffer_get_records_count(
  */
 static inline
 unsigned long subbuffer_count_records_overrun(
-				const struct lib_ring_buffer_config *config,
-				struct lib_ring_buffer_backend *bufb,
-				unsigned long idx)
+				const struct lttng_ust_lib_ring_buffer_config *config,
+				struct lttng_ust_lib_ring_buffer_backend *bufb,
+				unsigned long idx,
+				struct lttng_ust_shm_handle *handle)
 {
-	struct lib_ring_buffer_backend_pages *pages;
+	struct lttng_ust_lib_ring_buffer_backend_pages_shmp *pages;
 	unsigned long overruns, sb_bindex;
 
-	sb_bindex = subbuffer_id_get_index(config, bufb->buf_wsb[idx].id);
-	pages = bufb->array[sb_bindex];
-	overruns = v_read(config, &pages->records_unread);
-	v_set(config, &pages->records_unread,
-	      v_read(config, &pages->records_commit));
-	v_set(config, &pages->records_commit, 0);
+	sb_bindex = subbuffer_id_get_index(config, shmp_index(handle, bufb->buf_wsb, idx)->id);
+	pages = shmp_index(handle, bufb->array, sb_bindex);
+	overruns = v_read(config, &shmp(handle, pages->shmp)->records_unread);
+	v_set(config, &shmp(handle, pages->shmp)->records_unread,
+	      v_read(config, &shmp(handle, pages->shmp)->records_commit));
+	v_set(config, &shmp(handle, pages->shmp)->records_commit, 0);
 
 	return overruns;
 }
 
 static inline
-void subbuffer_set_data_size(const struct lib_ring_buffer_config *config,
-			     struct lib_ring_buffer_backend *bufb,
+void subbuffer_set_data_size(const struct lttng_ust_lib_ring_buffer_config *config,
+			     struct lttng_ust_lib_ring_buffer_backend *bufb,
 			     unsigned long idx,
-			     unsigned long data_size)
+			     unsigned long data_size,
+			     struct lttng_ust_shm_handle *handle)
 {
-	struct lib_ring_buffer_backend_pages *pages;
+	struct lttng_ust_lib_ring_buffer_backend_pages_shmp *pages;
 	unsigned long sb_bindex;
 
-	sb_bindex = subbuffer_id_get_index(config, bufb->buf_wsb[idx].id);
-	pages = bufb->array[sb_bindex];
-	pages->data_size = data_size;
+	sb_bindex = subbuffer_id_get_index(config, shmp_index(handle, bufb->buf_wsb, idx)->id);
+	pages = shmp_index(handle, bufb->array, sb_bindex);
+	shmp(handle, pages->shmp)->data_size = data_size;
 }
 
 static inline
 unsigned long subbuffer_get_read_data_size(
-				const struct lib_ring_buffer_config *config,
-				struct lib_ring_buffer_backend *bufb)
+				const struct lttng_ust_lib_ring_buffer_config *config,
+				struct lttng_ust_lib_ring_buffer_backend *bufb,
+				struct lttng_ust_shm_handle *handle)
 {
-	struct lib_ring_buffer_backend_pages *pages;
+	struct lttng_ust_lib_ring_buffer_backend_pages_shmp *pages;
 	unsigned long sb_bindex;
 
 	sb_bindex = subbuffer_id_get_index(config, bufb->buf_rsb.id);
-	pages = bufb->array[sb_bindex];
-	return pages->data_size;
+	pages = shmp_index(handle, bufb->array, sb_bindex);
+	return shmp(handle, pages->shmp)->data_size;
 }
 
 static inline
 unsigned long subbuffer_get_data_size(
-				const struct lib_ring_buffer_config *config,
-				struct lib_ring_buffer_backend *bufb,
-				unsigned long idx)
+				const struct lttng_ust_lib_ring_buffer_config *config,
+				struct lttng_ust_lib_ring_buffer_backend *bufb,
+				unsigned long idx,
+				struct lttng_ust_shm_handle *handle)
 {
-	struct lib_ring_buffer_backend_pages *pages;
+	struct lttng_ust_lib_ring_buffer_backend_pages_shmp *pages;
 	unsigned long sb_bindex;
 
-	sb_bindex = subbuffer_id_get_index(config, bufb->buf_wsb[idx].id);
-	pages = bufb->array[sb_bindex];
-	return pages->data_size;
+	sb_bindex = subbuffer_id_get_index(config, shmp_index(handle, bufb->buf_wsb, idx)->id);
+	pages = shmp_index(handle, bufb->array, sb_bindex);
+	return shmp(handle, pages->shmp)->data_size;
 }
 
 /**
@@ -290,9 +301,10 @@ unsigned long subbuffer_get_data_size(
  *                               writer.
  */
 static inline
-void lib_ring_buffer_clear_noref(const struct lib_ring_buffer_config *config,
-				 struct lib_ring_buffer_backend *bufb,
-				 unsigned long idx)
+void lib_ring_buffer_clear_noref(const struct lttng_ust_lib_ring_buffer_config *config,
+				 struct lttng_ust_lib_ring_buffer_backend *bufb,
+				 unsigned long idx,
+				 struct lttng_ust_shm_handle *handle)
 {
 	unsigned long id, new_id;
 
@@ -303,10 +315,10 @@ void lib_ring_buffer_clear_noref(const struct lib_ring_buffer_config *config,
 	 * Performing a volatile access to read the sb_pages, because we want to
 	 * read a coherent version of the pointer and the associated noref flag.
 	 */
-	id = CMM_ACCESS_ONCE(bufb->buf_wsb[idx].id);
+	id = CMM_ACCESS_ONCE(shmp_index(handle, bufb->buf_wsb, idx)->id);
 	for (;;) {
 		/* This check is called on the fast path for each record. */
-		if (likely(!subbuffer_id_is_noref(config, id))) {
+		if (caa_likely(!subbuffer_id_is_noref(config, id))) {
 			/*
 			 * Store after load dependency ordering the writes to
 			 * the subbuffer after load and test of the noref flag
@@ -317,8 +329,8 @@ void lib_ring_buffer_clear_noref(const struct lib_ring_buffer_config *config,
 		}
 		new_id = id;
 		subbuffer_id_clear_noref(config, &new_id);
-		new_id = uatomic_cmpxchg(&bufb->buf_wsb[idx].id, id, new_id);
-		if (likely(new_id == id))
+		new_id = uatomic_cmpxchg(&shmp_index(handle, bufb->buf_wsb, idx)->id, id, new_id);
+		if (caa_likely(new_id == id))
 			break;
 		id = new_id;
 	}
@@ -329,9 +341,10 @@ void lib_ring_buffer_clear_noref(const struct lib_ring_buffer_config *config,
  *                                    called by writer.
  */
 static inline
-void lib_ring_buffer_set_noref_offset(const struct lib_ring_buffer_config *config,
-				      struct lib_ring_buffer_backend *bufb,
-				      unsigned long idx, unsigned long offset)
+void lib_ring_buffer_set_noref_offset(const struct lttng_ust_lib_ring_buffer_config *config,
+				      struct lttng_ust_lib_ring_buffer_backend *bufb,
+				      unsigned long idx, unsigned long offset,
+				      struct lttng_ust_shm_handle *handle)
 {
 	if (config->mode != RING_BUFFER_OVERWRITE)
 		return;
@@ -347,25 +360,26 @@ void lib_ring_buffer_set_noref_offset(const struct lib_ring_buffer_config *confi
 	 * subbuffer_set_noref() uses a volatile store to deal with concurrent
 	 * readers of the noref flag.
 	 */
-	CHAN_WARN_ON(bufb->chan,
-		     subbuffer_id_is_noref(config, bufb->buf_wsb[idx].id));
+	CHAN_WARN_ON(shmp(handle, bufb->chan),
+		     subbuffer_id_is_noref(config, shmp_index(handle, bufb->buf_wsb, idx)->id));
 	/*
 	 * Memory barrier that ensures counter stores are ordered before set
 	 * noref and offset.
 	 */
 	cmm_smp_mb();
-	subbuffer_id_set_noref_offset(config, &bufb->buf_wsb[idx].id, offset);
+	subbuffer_id_set_noref_offset(config, &shmp_index(handle, bufb->buf_wsb, idx)->id, offset);
 }
 
 /**
  * update_read_sb_index - Read-side subbuffer index update.
  */
 static inline
-int update_read_sb_index(const struct lib_ring_buffer_config *config,
-			 struct lib_ring_buffer_backend *bufb,
+int update_read_sb_index(const struct lttng_ust_lib_ring_buffer_config *config,
+			 struct lttng_ust_lib_ring_buffer_backend *bufb,
 			 struct channel_backend *chanb,
 			 unsigned long consumed_idx,
-			 unsigned long consumed_count)
+			 unsigned long consumed_count,
+			 struct lttng_ust_shm_handle *handle)
 {
 	unsigned long old_id, new_id;
 
@@ -376,28 +390,28 @@ int update_read_sb_index(const struct lib_ring_buffer_config *config,
 		 * old_wpage, because the value read will be confirmed by the
 		 * following cmpxchg().
 		 */
-		old_id = bufb->buf_wsb[consumed_idx].id;
-		if (unlikely(!subbuffer_id_is_noref(config, old_id)))
+		old_id = shmp_index(handle, bufb->buf_wsb, consumed_idx)->id;
+		if (caa_unlikely(!subbuffer_id_is_noref(config, old_id)))
 			return -EAGAIN;
 		/*
 		 * Make sure the offset count we are expecting matches the one
 		 * indicated by the writer.
 		 */
-		if (unlikely(!subbuffer_id_compare_offset(config, old_id,
+		if (caa_unlikely(!subbuffer_id_compare_offset(config, old_id,
 							  consumed_count)))
 			return -EAGAIN;
-		CHAN_WARN_ON(bufb->chan,
+		CHAN_WARN_ON(shmp(handle, bufb->chan),
 			     !subbuffer_id_is_noref(config, bufb->buf_rsb.id));
 		subbuffer_id_set_noref_offset(config, &bufb->buf_rsb.id,
 					      consumed_count);
-		new_id = uatomic_cmpxchg(&bufb->buf_wsb[consumed_idx].id, old_id,
+		new_id = uatomic_cmpxchg(&shmp_index(handle, bufb->buf_wsb, consumed_idx)->id, old_id,
 				 bufb->buf_rsb.id);
-		if (unlikely(old_id != new_id))
+		if (caa_unlikely(old_id != new_id))
 			return -EAGAIN;
 		bufb->buf_rsb.id = new_id;
 	} else {
 		/* No page exchange, use the writer page directly */
-		bufb->buf_rsb.id = bufb->buf_wsb[consumed_idx].id;
+		bufb->buf_rsb.id = shmp_index(handle, bufb->buf_wsb, consumed_idx)->id;
 	}
 	return 0;
 }