[urcu.git] / rcuja / rcuja-internal.h

#ifndef _URCU_RCUJA_INTERNAL_H
#define _URCU_RCUJA_INTERNAL_H

/*
 * rcuja/rcuja-internal.h
 *
 * Userspace RCU library - RCU Judy Array Internal Header
 *
 * Copyright 2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */

#define _GNU_SOURCE
#include <pthread.h>
#include <stdio.h>
#include <inttypes.h>
#include <unistd.h>
#include <urcu/rculfhash.h>

/*
 * Number of least significant pointer bits reserved to represent the
 * child type.
 */
#define JA_TYPE_BITS	3
#define JA_TYPE_MAX_NR	(1UL << JA_TYPE_BITS)
#define JA_TYPE_MASK	(JA_TYPE_MAX_NR - 1)
#define JA_PTR_MASK	(~JA_TYPE_MASK)

#define JA_ENTRY_PER_NODE	256UL
#define JA_LOG2_BITS_PER_BYTE	3U
#define JA_BITS_PER_BYTE	(1U << JA_LOG2_BITS_PER_BYTE)

#define JA_POOL_1D_MASK	((JA_BITS_PER_BYTE - 1) << JA_TYPE_BITS)
#define JA_POOL_2D_MASK	(JA_POOL_1D_MASK << JA_LOG2_BITS_PER_BYTE)

#define JA_MAX_DEPTH	9	/* Maximum depth, including leafs */

/*
 * Entry for NULL node is at index 8 of the table. It is never encoded
 * in flags.
 */
#define NODE_INDEX_NULL		8

/*
 * Number of removals needed on a fallback node before we try to shrink
 * it.
 */
#define JA_FALLBACK_REMOVAL_COUNT	8

/* Never declared. Opaque type used to store flagged node pointers. */
struct cds_ja_inode_flag;
struct cds_ja_inode;

/*
 * Shadow node contains mutex and call_rcu head associated with a node.
 */
struct cds_ja_shadow_node {
	struct cds_lfht_node ht_node;	/* hash table node */
	struct cds_ja_inode_flag *node_flag;	/* reverse mapping and hash table key */
	/*
	 * mutual exclusion on all nodes belonging to the same tree
	 * position (e.g. both nodes before and after recompaction
	 * use the same lock).
	 */
	pthread_mutex_t *lock;
	unsigned int nr_child;		/* number of children in node */
	struct rcu_head head;		/* for deferred node and shadow node reclaim */
	int fallback_removal_count;	/* removals left keeping fallback */
	int level;			/* level in the tree */
	struct cds_ja *ja;		/* toplevel judy array */
};

struct cds_ja {
	struct cds_ja_inode_flag *root;
	unsigned int tree_depth;
	uint64_t key_max;
	/*
	 * We use a hash table to associate node keys to their
	 * respective shadow node. This helps reducing lookup hot path
	 * cache footprint, especially for very small nodes.
	 */
	struct cds_lfht *ht;
	unsigned long nr_fallback;	/* Number of fallback nodes used */
};

static inline
struct cds_ja_inode_flag *ja_node_flag(struct cds_ja_inode *node,
		unsigned long type)
{
	assert(type < (1UL << JA_TYPE_BITS));
	return (struct cds_ja_inode_flag *) (((unsigned long) node) | type);
}

static inline
struct cds_ja_inode_flag *ja_node_flag_pool_1d(struct cds_ja_inode *node,
		unsigned long type, unsigned long bitsel)
{
	assert(type < (1UL << JA_TYPE_BITS));
	assert(bitsel < JA_BITS_PER_BYTE);
	return (struct cds_ja_inode_flag *) (((unsigned long) node) | (bitsel << JA_TYPE_BITS) | type);
}

static inline
struct cds_ja_inode_flag *ja_node_flag_pool_2d(struct cds_ja_inode *node,
		unsigned long type, unsigned int bitsel[2])
{
	assert(type < (1UL << JA_TYPE_BITS));
	assert(bitsel[0] < JA_BITS_PER_BYTE);
	assert(bitsel[1] < JA_BITS_PER_BYTE);
	return (struct cds_ja_inode_flag *) (((unsigned long) node) | (bitsel[0] << (JA_TYPE_BITS + JA_LOG2_BITS_PER_BYTE)) | (bitsel[1] << JA_TYPE_BITS) | type);
}

static inline
unsigned long ja_node_pool_1d_bitsel(struct cds_ja_inode_flag *node)
{
	return ((unsigned long) node & JA_POOL_1D_MASK) >> JA_TYPE_BITS;
}

static inline
void ja_node_pool_2d_bitsel(struct cds_ja_inode_flag *node, unsigned long *bits)
{
	bits[0] = ((unsigned long) node & JA_POOL_2D_MASK) >> (JA_TYPE_BITS + JA_LOG2_BITS_PER_BYTE);
	bits[1] = ((unsigned long) node & JA_POOL_1D_MASK) >> JA_TYPE_BITS;
}

__attribute__((visibility("protected")))
unsigned long ja_node_type(struct cds_ja_inode_flag *node);

__attribute__((visibility("protected")))
struct cds_ja_inode *ja_node_ptr(struct cds_ja_inode_flag *node);

__attribute__((visibility("protected")))
void rcuja_free_all_children(struct cds_ja_shadow_node *shadow_node,
		struct cds_ja_inode_flag *node_flag,
		void (*free_node_cb)(struct rcu_head *head));

__attribute__((visibility("protected")))
struct cds_ja_shadow_node *rcuja_shadow_lookup_lock(struct cds_lfht *ht,
		struct cds_ja_inode_flag *node_flag);

__attribute__((visibility("protected")))
void rcuja_shadow_unlock(struct cds_ja_shadow_node *shadow_node);

__attribute__((visibility("protected")))
struct cds_ja_shadow_node *rcuja_shadow_set(struct cds_lfht *ht,
		struct cds_ja_inode_flag *new_node_flag,
		struct cds_ja_shadow_node *inherit_from,
		struct cds_ja *ja, int level);

/* rcuja_shadow_clear flags */
enum {
	RCUJA_SHADOW_CLEAR_FREE_NODE = (1U << 0),
	RCUJA_SHADOW_CLEAR_FREE_LOCK = (1U << 1),
};

__attribute__((visibility("protected")))
int rcuja_shadow_clear(struct cds_lfht *ht,
		struct cds_ja_inode_flag *node_flag,
		struct cds_ja_shadow_node *shadow_node,
		unsigned int flags);

__attribute__((visibility("protected")))
void rcuja_shadow_prune(struct cds_lfht *ht,
		unsigned int flags,
		void (*free_node_cb)(struct rcu_head *head));

__attribute__((visibility("protected")))
struct cds_lfht *rcuja_create_ht(const struct rcu_flavor_struct *flavor);

__attribute__((visibility("protected")))
int rcuja_delete_ht(struct cds_lfht *ht);

__attribute__((visibility("protected")))
void free_cds_ja_node(struct cds_ja_inode *node);

//#define DEBUG

#ifdef __linux__
#include <syscall.h>
#endif

#if defined(_syscall0)
_syscall0(pid_t, gettid)
#elif defined(__NR_gettid)
static inline pid_t gettid(void)
{
	return syscall(__NR_gettid);
}
#else
#warning "use pid as tid"
static inline pid_t gettid(void)
{
	return getpid();
}
#endif

#ifdef DEBUG
#define dbg_printf(fmt, args...)				\
	fprintf(stderr, "[debug rcuja %lu %s()@%s:%u] " fmt,	\
		(unsigned long) gettid(), __func__,		\
		__FILE__, __LINE__, ## args)
#else
#define dbg_printf(fmt, args...)				\
do {								\
	/* do nothing but check printf format */		\
	if (0)							\
		fprintf(stderr, "[debug rcuja %lu %s()@%s:%u] " fmt, \
			(unsigned long) gettid(), __func__,	\
			__FILE__, __LINE__, ## args);		\
} while (0)
#endif

#endif /* _URCU_RCUJA_INTERNAL_H */
Commit	Line	Data
61009379 MD	1	#ifndef _URCU_RCUJA_INTERNAL_H
	2	#define _URCU_RCUJA_INTERNAL_H
	3
	4	/*
	5	* rcuja/rcuja-internal.h
	6	*
	7	* Userspace RCU library - RCU Judy Array Internal Header
	8	*
	9	* Copyright 2012 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	10	*
	11	* This library is free software; you can redistribute it and/or
	12	* modify it under the terms of the GNU Lesser General Public
	13	* License as published by the Free Software Foundation; either
	14	* version 2.1 of the License, or (at your option) any later version.
	15	*
	16	* This library is distributed in the hope that it will be useful,
	17	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	18	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	19	* Lesser General Public License for more details.
	20	*
	21	* You should have received a copy of the GNU Lesser General Public
	22	* License along with this library; if not, write to the Free Software
	23	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	24	*/
	25
7d67da99	26	#define _GNU_SOURCE
511b4dcc	27	#include <pthread.h>
a2a7ff59 MD	28	#include <stdio.h>
a2a7ff59 MD	29	#include <inttypes.h>
7d67da99	30	#include <unistd.h>
511b4dcc MD	31	#include <urcu/rculfhash.h>
511b4dcc MD	32
3d8fe307 MD	33	/*
	34	* Number of least significant pointer bits reserved to represent the
	35	* child type.
	36	*/
	37	#define JA_TYPE_BITS 3
	38	#define JA_TYPE_MAX_NR (1UL << JA_TYPE_BITS)
	39	#define JA_TYPE_MASK (JA_TYPE_MAX_NR - 1)
	40	#define JA_PTR_MASK (~JA_TYPE_MASK)
	41
	42	#define JA_ENTRY_PER_NODE 256UL
	43	#define JA_LOG2_BITS_PER_BYTE 3U
	44	#define JA_BITS_PER_BYTE (1U << JA_LOG2_BITS_PER_BYTE)
	45
b1a90ce3 MD	46	#define JA_POOL_1D_MASK ((JA_BITS_PER_BYTE - 1) << JA_TYPE_BITS)
	47	#define JA_POOL_2D_MASK (JA_POOL_1D_MASK << JA_LOG2_BITS_PER_BYTE)
	48
3d8fe307 MD	49	#define JA_MAX_DEPTH 9 /* Maximum depth, including leafs */
	50
	51	/*
	52	* Entry for NULL node is at index 8 of the table. It is never encoded
	53	* in flags.
	54	*/
	55	#define NODE_INDEX_NULL 8
	56
	57	/*
	58	* Number of removals needed on a fallback node before we try to shrink
	59	* it.
	60	*/
	61	#define JA_FALLBACK_REMOVAL_COUNT 8
	62
511b4dcc	63	/* Never declared. Opaque type used to store flagged node pointers. */
b4540e8a	64	struct cds_ja_inode_flag;
3d8fe307	65	struct cds_ja_inode;
511b4dcc MD	66
	67	/*
	68	* Shadow node contains mutex and call_rcu head associated with a node.
	69	*/
d96bfb0d	70	struct cds_ja_shadow_node {
d22c185b	71	struct cds_lfht_node ht_node; /* hash table node */
3d8fe307	72	struct cds_ja_inode_flag node_flag; / reverse mapping and hash table key */
e1db2db5 MD	73	/*
	74	* mutual exclusion on all nodes belonging to the same tree
	75	* position (e.g. both nodes before and after recompaction
	76	* use the same lock).
	77	*/
	78	pthread_mutex_t *lock;
	79	unsigned int nr_child; /* number of children in node */
d22c185b	80	struct rcu_head head; /* for deferred node and shadow node reclaim */
f07b240f	81	int fallback_removal_count; /* removals left keeping fallback */
3d8fe307 MD	82	int level; /* level in the tree */
3d8fe307 MD	83	struct cds_ja ja; / toplevel judy array */
511b4dcc MD	84	};
511b4dcc MD	85
d96bfb0d	86	struct cds_ja {
b4540e8a MD	87	struct cds_ja_inode_flag *root;
	88	unsigned int tree_depth;
	89	uint64_t key_max;
511b4dcc	90	/*
e1db2db5 MD	91	* We use a hash table to associate node keys to their
	92	* respective shadow node. This helps reducing lookup hot path
	93	* cache footprint, especially for very small nodes.
511b4dcc MD	94	*/
511b4dcc MD	95	struct cds_lfht *ht;
f07b240f	96	unsigned long nr_fallback; /* Number of fallback nodes used */
511b4dcc	97	};
61009379	98
3d8fe307 MD	99	static inline
	100	struct cds_ja_inode_flag ja_node_flag(struct cds_ja_inode node,
	101	unsigned long type)
	102	{
	103	assert(type < (1UL << JA_TYPE_BITS));
	104	return (struct cds_ja_inode_flag *) (((unsigned long) node) \| type);
	105	}
	106
	107	static inline
b1a90ce3 MD	108	struct cds_ja_inode_flag ja_node_flag_pool_1d(struct cds_ja_inode node,
b1a90ce3 MD	109	unsigned long type, unsigned long bitsel)
3d8fe307	110	{
b1a90ce3 MD	111	assert(type < (1UL << JA_TYPE_BITS));
	112	assert(bitsel < JA_BITS_PER_BYTE);
	113	return (struct cds_ja_inode_flag *) (((unsigned long) node) \| (bitsel << JA_TYPE_BITS) \| type);
3d8fe307 MD	114	}
3d8fe307 MD	115
19ddcd04 MD	116	static inline
	117	struct cds_ja_inode_flag ja_node_flag_pool_2d(struct cds_ja_inode node,
	118	unsigned long type, unsigned int bitsel[2])
	119	{
	120	assert(type < (1UL << JA_TYPE_BITS));
	121	assert(bitsel[0] < JA_BITS_PER_BYTE);
	122	assert(bitsel[1] < JA_BITS_PER_BYTE);
	123	return (struct cds_ja_inode_flag *) (((unsigned long) node) \| (bitsel[0] << (JA_TYPE_BITS + JA_LOG2_BITS_PER_BYTE)) \| (bitsel[1] << JA_TYPE_BITS) \| type);
	124	}
	125
3d8fe307	126	static inline
b1a90ce3	127	unsigned long ja_node_pool_1d_bitsel(struct cds_ja_inode_flag *node)
3d8fe307	128	{
b1a90ce3 MD	129	return ((unsigned long) node & JA_POOL_1D_MASK) >> JA_TYPE_BITS;
b1a90ce3 MD	130	}
3d8fe307	131
b1a90ce3 MD	132	static inline
	133	void ja_node_pool_2d_bitsel(struct cds_ja_inode_flag node, unsigned long bits)
	134	{
	135	bits[0] = ((unsigned long) node & JA_POOL_2D_MASK) >> (JA_TYPE_BITS + JA_LOG2_BITS_PER_BYTE);
	136	bits[1] = ((unsigned long) node & JA_POOL_1D_MASK) >> JA_TYPE_BITS;
3d8fe307 MD	137	}
3d8fe307 MD	138
b1a90ce3 MD	139	__attribute__((visibility("protected")))
	140	unsigned long ja_node_type(struct cds_ja_inode_flag *node);
	141
	142	__attribute__((visibility("protected")))
	143	struct cds_ja_inode ja_node_ptr(struct cds_ja_inode_flag node);
	144
3d8fe307 MD	145	__attribute__((visibility("protected")))
	146	void rcuja_free_all_children(struct cds_ja_shadow_node *shadow_node,
	147	struct cds_ja_inode_flag *node_flag,
	148	void (free_node_cb)(struct rcu_head head));
	149
25fde237	150	__attribute__((visibility("protected")))
d96bfb0d	151	struct cds_ja_shadow_node rcuja_shadow_lookup_lock(struct cds_lfht ht,
3d8fe307	152	struct cds_ja_inode_flag *node_flag);
be9a7474	153
25fde237	154	__attribute__((visibility("protected")))
d96bfb0d	155	void rcuja_shadow_unlock(struct cds_ja_shadow_node *shadow_node);
be9a7474	156
25fde237	157	__attribute__((visibility("protected")))
f07b240f	158	struct cds_ja_shadow_node rcuja_shadow_set(struct cds_lfht ht,
3d8fe307 MD	159	struct cds_ja_inode_flag *new_node_flag,
3d8fe307 MD	160	struct cds_ja_shadow_node *inherit_from,
48cbe001	161	struct cds_ja *ja, int level);
e1db2db5 MD	162
	163	/* rcuja_shadow_clear flags */
	164	enum {
	165	RCUJA_SHADOW_CLEAR_FREE_NODE = (1U << 0),
	166	RCUJA_SHADOW_CLEAR_FREE_LOCK = (1U << 1),
	167	};
	168
be9a7474	169	__attribute__((visibility("protected")))
e1db2db5	170	int rcuja_shadow_clear(struct cds_lfht *ht,
3d8fe307	171	struct cds_ja_inode_flag *node_flag,
a2a7ff59	172	struct cds_ja_shadow_node *shadow_node,
e1db2db5	173	unsigned int flags);
be9a7474 MD	174
	175	__attribute__((visibility("protected")))
	176	void rcuja_shadow_prune(struct cds_lfht *ht,
3d8fe307 MD	177	unsigned int flags,
3d8fe307 MD	178	void (free_node_cb)(struct rcu_head head));
be9a7474	179
25fde237	180	__attribute__((visibility("protected")))
5eb692c0	181	struct cds_lfht rcuja_create_ht(const struct rcu_flavor_struct flavor);
be9a7474	182
25fde237	183	__attribute__((visibility("protected")))
be9a7474	184	int rcuja_delete_ht(struct cds_lfht *ht);
25fde237	185
48cbe001 MD	186	__attribute__((visibility("protected")))
	187	void free_cds_ja_node(struct cds_ja_inode *node);
	188
1a0c0717	189	//#define DEBUG
a2a7ff59	190
7d67da99 MD	191	#ifdef __linux__
	192	#include <syscall.h>
	193	#endif
	194
	195	#if defined(_syscall0)
	196	_syscall0(pid_t, gettid)
	197	#elif defined(__NR_gettid)
	198	static inline pid_t gettid(void)
	199	{
	200	return syscall(__NR_gettid);
	201	}
	202	#else
	203	#warning "use pid as tid"
	204	static inline pid_t gettid(void)
	205	{
	206	return getpid();
	207	}
	208	#endif
	209
a2a7ff59	210	#ifdef DEBUG
7d67da99 MD	211	#define dbg_printf(fmt, args...) \
	212	fprintf(stderr, "[debug rcuja %lu %s()@%s:%u] " fmt, \
	213	(unsigned long) gettid(), __func__, \
	214	__FILE__, __LINE__, ## args)
a2a7ff59 MD	215	#else
	216	#define dbg_printf(fmt, args...) \
	217	do { \
	218	/* do nothing but check printf format */ \
	219	if (0) \
7d67da99 MD	220	fprintf(stderr, "[debug rcuja %lu %s()@%s:%u] " fmt, \
	221	(unsigned long) gettid(), __func__, \
	222	__FILE__, __LINE__, ## args); \
a2a7ff59 MD	223	} while (0)
	224	#endif
	225
61009379	226	#endif /* _URCU_RCUJA_INTERNAL_H */