From: Mathieu Desnoyers Date: Sat, 2 Jul 2011 15:31:09 +0000 (-0400) Subject: Rename urcu-ht to rculfhash X-Git-Tag: v0.7.0~43^2~234 X-Git-Url: https://git.lttng.org/?p=urcu.git;a=commitdiff_plain;h=a42cc6594389e8f88cc49e36a7779afdb6529135 Rename urcu-ht to rculfhash Signed-off-by: Mathieu Desnoyers --- diff --git a/Makefile.am b/Makefile.am index 5869c5e..ef11058 100644 --- a/Makefile.am +++ b/Makefile.am @@ -14,7 +14,8 @@ nobase_dist_include_HEADERS = urcu/compiler.h urcu/hlist.h urcu/list.h \ urcu/uatomic/generic.h urcu/arch/generic.h urcu/wfstack.h \ urcu/wfqueue.h urcu/rculfstack.h urcu/rculfqueue.h \ urcu/ref.h urcu/map/*.h urcu/static/*.h urcu/cds.h \ - urcu/urcu_ref.h urcu/urcu-futex.h urcu/uatomic_arch.h + urcu/urcu_ref.h urcu/urcu-futex.h urcu/uatomic_arch.h \ + urcu/urcu-ht.h nobase_nodist_include_HEADERS = urcu/arch.h urcu/uatomic.h urcu/config.h EXTRA_DIST = $(top_srcdir)/urcu/arch/*.h $(top_srcdir)/urcu/uatomic/*.h \ @@ -37,7 +38,7 @@ lib_LTLIBRARIES = liburcu-cds.la liburcu.la liburcu-qsbr.la \ liburcu-mb.la liburcu-signal.la liburcu-bp.la liburcu_cds_la_SOURCES = wfqueue.c wfstack.c rculfqueue.c rculfstack.c \ - $(COMPAT) + rculfhash.c $(COMPAT) liburcu_la_SOURCES = urcu.c urcu-pointer.c $(COMPAT) liburcu_la_LIBADD = liburcu-cds.la diff --git a/rculfhash.c b/rculfhash.c new file mode 100644 index 0000000..ec799dc --- /dev/null +++ b/rculfhash.c @@ -0,0 +1,485 @@ + +/* + * TODO: keys are currently assumed <= sizeof(void *). Key target never freed. + */ + +#define _LGPL_SOURCE +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * Maximum number of hash table buckets: 256M on 64-bit. + * Should take about 512MB max if we assume 1 node per 4 buckets. + */ +#define MAX_HT_BUCKETS ((256 << 10) / sizeof(void *)) + +/* node flags */ +#define NODE_STOLEN (1 << 0) + +struct rcu_ht_node; + +struct rcu_ht_node { + struct rcu_ht_node *next; + void *key; + void *data; + unsigned int flags; +}; + +struct rcu_table { + unsigned long size; + struct rcu_ht_node *tbl[0]; +}; + +struct rcu_ht { + struct rcu_table *t; /* shared */ + ht_hash_fct hash_fct; + void (*free_fct)(void *data); /* fct to free data */ + uint32_t keylen; + uint32_t hashseed; + pthread_mutex_t resize_mutex; /* resize mutex: add/del mutex */ + int resize_ongoing; /* fast-path resize check */ +}; + +struct rcu_ht *ht_new(ht_hash_fct hash_fct, void (*free_fct)(void *data), + unsigned long init_size, uint32_t keylen, + uint32_t hashseed) +{ + struct rcu_ht *ht; + + ht = calloc(1, sizeof(struct rcu_ht)); + ht->hash_fct = hash_fct; + ht->free_fct = free_fct; + ht->keylen = keylen; + ht->hashseed = hashseed; + /* this mutex should not nest in read-side C.S. */ + pthread_mutex_init(&ht->resize_mutex, NULL); + ht->resize_ongoing = 0; /* shared */ + ht->t = calloc(1, sizeof(struct rcu_table) + + (init_size * sizeof(struct rcu_ht_node *))); + ht->t->size = init_size; + return ht; +} + +void *ht_lookup(struct rcu_ht *ht, void *key) +{ + struct rcu_table *t; + unsigned long hash; + struct rcu_ht_node *node; + void *ret; + + rcu_read_lock(); + t = rcu_dereference(ht->t); + smp_read_barrier_depends(); /* read t before size and table */ + hash = ht->hash_fct(key, ht->keylen, ht->hashseed) % t->size; + smp_read_barrier_depends(); /* read size before links */ + node = rcu_dereference(t->tbl[hash]); + for (;;) { + if (likely(!node)) { + ret = NULL; + break; + } + if (node->key == key) { + ret = node->data; + break; + } + node = rcu_dereference(node->next); + } + rcu_read_unlock(); + + return ret; +} + +/* + * Will re-try until either: + * - The key is already there (-EEXIST) + * - We successfully add the key at the head of a table bucket. + */ +int ht_add(struct rcu_ht *ht, void *key, void *data) +{ + struct rcu_ht_node *node, *old_head, *new_head; + struct rcu_table *t; + unsigned long hash; + int ret = 0; + + new_head = calloc(1, sizeof(struct rcu_ht_node)); + new_head->key = key; + new_head->data = data; + new_head->flags = 0; + /* here comes the fun and tricky part. + * Add at the beginning with a cmpxchg. + * Hold a read lock between the moment the first element is read + * and the nodes traversal (to find duplicates). This ensures + * the head pointer has not been reclaimed when cmpxchg is done. + * Always adding at the head ensures that we would have to + * re-try if a new item has been added concurrently. So we ensure that + * we never add duplicates. */ +retry: + rcu_read_lock(); + + if (unlikely(LOAD_SHARED(ht->resize_ongoing))) { + rcu_read_unlock(); + /* + * Wait for resize to complete before continuing. + */ + ret = pthread_mutex_lock(&ht->resize_mutex); + assert(!ret); + ret = pthread_mutex_unlock(&ht->resize_mutex); + assert(!ret); + goto retry; + } + + t = rcu_dereference(ht->t); + /* no read barrier needed, because no concurrency with resize */ + hash = ht->hash_fct(key, ht->keylen, ht->hashseed) % t->size; + + old_head = node = rcu_dereference(t->tbl[hash]); + for (;;) { + if (likely(!node)) { + break; + } + if (node->key == key) { + ret = -EEXIST; + goto end; + } + node = rcu_dereference(node->next); + } + new_head->next = old_head; + if (rcu_cmpxchg_pointer(&t->tbl[hash], old_head, new_head) != old_head) + goto restart; +end: + rcu_read_unlock(); + return ret; + + /* restart loop, release and re-take the read lock to be kind to GP */ +restart: + rcu_read_unlock(); + goto retry; +} + +/* + * Restart until we successfully remove the entry, or no entry is left + * ((void *)(unsigned long)-ENOENT). + * Deal with concurrent stealers by doing an extra verification pass to check + * that no element in the list are still pointing to the element stolen. + * This could happen if two concurrent steal for consecutive objects are + * executed. A pointer to an object being stolen could be saved by the + * concurrent stealer for the previous object. + * Also, given that in this precise scenario, another stealer can also want to + * delete the doubly-referenced object; use a "stolen" flag to let only one + * stealer delete the object. + */ +void *ht_steal(struct rcu_ht *ht, void *key) +{ + struct rcu_ht_node **prev, *node, *del_node = NULL; + struct rcu_table *t; + unsigned long hash; + void *data; + int ret; + +retry: + rcu_read_lock(); + + if (unlikely(LOAD_SHARED(ht->resize_ongoing))) { + rcu_read_unlock(); + /* + * Wait for resize to complete before continuing. + */ + ret = pthread_mutex_lock(&ht->resize_mutex); + assert(!ret); + ret = pthread_mutex_unlock(&ht->resize_mutex); + assert(!ret); + goto retry; + } + + t = rcu_dereference(ht->t); + /* no read barrier needed, because no concurrency with resize */ + hash = ht->hash_fct(key, ht->keylen, ht->hashseed) % t->size; + + prev = &t->tbl[hash]; + node = rcu_dereference(*prev); + for (;;) { + if (likely(!node)) { + if (del_node) { + goto end; + } else { + goto error; + } + } + if (node->key == key) { + break; + } + prev = &node->next; + node = rcu_dereference(*prev); + } + + if (!del_node) { + /* + * Another concurrent thread stole it ? If so, let it deal with + * this. Assume NODE_STOLEN is the only flag. If this changes, + * read flags before cmpxchg. + */ + if (cmpxchg(&node->flags, 0, NODE_STOLEN) != 0) + goto error; + } + + /* Found it ! pointer to object is in "prev" */ + if (rcu_cmpxchg_pointer(prev, node, node->next) == node) + del_node = node; + goto restart; + +end: + /* + * From that point, we own node. Note that there can still be concurrent + * RCU readers using it. We can free it outside of read lock after a GP. + */ + rcu_read_unlock(); + + data = del_node->data; + call_rcu(free, del_node); + return data; + +error: + data = (void *)(unsigned long)-ENOENT; + rcu_read_unlock(); + return data; + + /* restart loop, release and re-take the read lock to be kind to GP */ +restart: + rcu_read_unlock(); + goto retry; +} + +int ht_delete(struct rcu_ht *ht, void *key) +{ + void *data; + + data = ht_steal(ht, key); + if (data && data != (void *)(unsigned long)-ENOENT) { + if (ht->free_fct) + call_rcu(ht->free_fct, data); + return 0; + } else { + return -ENOENT; + } +} + +/* Delete all old elements. Allow concurrent writer accesses. */ +int ht_delete_all(struct rcu_ht *ht) +{ + unsigned long i; + struct rcu_ht_node **prev, *node, *inext; + struct rcu_table *t; + int cnt = 0; + int ret; + + /* + * Mutual exclusion with resize operations, but leave add/steal execute + * concurrently. This is OK because we operate only on the heads. + */ + ret = pthread_mutex_lock(&ht->resize_mutex); + assert(!ret); + + t = rcu_dereference(ht->t); + /* no read barrier needed, because no concurrency with resize */ + for (i = 0; i < t->size; i++) { + rcu_read_lock(); + prev = &t->tbl[i]; + /* + * Cut the head. After that, we own the first element. + */ + node = rcu_xchg_pointer(prev, NULL); + if (!node) { + rcu_read_unlock(); + continue; + } + /* + * We manage a list shared with concurrent writers and readers. + * Note that a concurrent add may or may not be deleted by us, + * depending if it arrives before or after the head is cut. + * "node" points to our first node. Remove first elements + * iteratively. + */ + for (;;) { + inext = NULL; + prev = &node->next; + if (prev) + inext = rcu_xchg_pointer(prev, NULL); + /* + * "node" is the first element of the list we have cut. + * We therefore own it, no concurrent writer may delete + * it. There can only be concurrent lookups. Concurrent + * add can only be done on a bucket head, but we've cut + * it already. inext is also owned by us, because we + * have exchanged it for "NULL". It will therefore be + * safe to use it after a G.P. + */ + rcu_read_unlock(); + if (node->data) + call_rcu(ht->free_fct, node->data); + call_rcu(free, node); + cnt++; + if (likely(!inext)) + break; + rcu_read_lock(); + node = inext; + } + } + + ret = pthread_mutex_unlock(&ht->resize_mutex); + assert(!ret); + return cnt; +} + +/* + * Should only be called when no more concurrent readers nor writers can + * possibly access the table. + */ +int ht_destroy(struct rcu_ht *ht) +{ + int ret; + + ret = ht_delete_all(ht); + free(ht->t); + free(ht); + return ret; +} + +static void ht_resize_grow(struct rcu_ht *ht) +{ + unsigned long i, new_size, old_size; + struct rcu_table *new_t, *old_t; + struct rcu_ht_node *node, *new_node, *tmp; + unsigned long hash; + + old_t = ht->t; + old_size = old_t->size; + + if (old_size == MAX_HT_BUCKETS) + return; + + new_size = old_size << 1; + new_t = calloc(1, sizeof(struct rcu_table) + + (new_size * sizeof(struct rcu_ht_node *))); + new_t->size = new_size; + + for (i = 0; i < old_size; i++) { + /* + * Re-hash each entry, insert in new table. + * It's important that a reader looking for a key _will_ find it + * if it's in the table. + * Copy each node. (just the node, not ->data) + */ + node = old_t->tbl[i]; + while (node) { + hash = ht->hash_fct(node->key, ht->keylen, ht->hashseed) + % new_size; + new_node = malloc(sizeof(struct rcu_ht_node)); + new_node->key = node->key; + new_node->data = node->data; + new_node->flags = node->flags; + new_node->next = new_t->tbl[hash]; /* link to first */ + new_t->tbl[hash] = new_node; /* add to head */ + node = node->next; + } + } + + /* Changing table and size atomically wrt lookups */ + rcu_assign_pointer(ht->t, new_t); + + /* Ensure all concurrent lookups use new size and table */ + synchronize_rcu(); + + for (i = 0; i < old_size; i++) { + node = old_t->tbl[i]; + while (node) { + tmp = node->next; + free(node); + node = tmp; + } + } + free(old_t); +} + +static void ht_resize_shrink(struct rcu_ht *ht) +{ + unsigned long i, new_size; + struct rcu_table *new_t, *old_t; + struct rcu_ht_node **prev, *node; + + old_t = ht->t; + if (old_t->size == 1) + return; + + new_size = old_t->size >> 1; + + for (i = 0; i < new_size; i++) { + /* Link end with first entry of i + new_size */ + prev = &old_t->tbl[i]; + node = *prev; + while (node) { + prev = &node->next; + node = *prev; + } + *prev = old_t->tbl[i + new_size]; + } + smp_wmb(); /* write links before changing size */ + STORE_SHARED(old_t->size, new_size); + + /* Ensure all concurrent lookups use new size */ + synchronize_rcu(); + + new_t = realloc(old_t, sizeof(struct rcu_table) + + (new_size * sizeof(struct rcu_ht_node *))); + /* shrinking, pointers should not move */ + assert(new_t == old_t); +} + +/* + * growth: >0: *2, <0: /2 + */ +void ht_resize(struct rcu_ht *ht, int growth) +{ + int ret; + + ret = pthread_mutex_lock(&ht->resize_mutex); + assert(!ret); + STORE_SHARED(ht->resize_ongoing, 1); + synchronize_rcu(); + /* All add/remove are waiting on the mutex. */ + if (growth > 0) + ht_resize_grow(ht); + else if (growth < 0) + ht_resize_shrink(ht); + smp_mb(); + STORE_SHARED(ht->resize_ongoing, 0); + ret = pthread_mutex_unlock(&ht->resize_mutex); + assert(!ret); +} + +/* + * Expects keys <= than pointer size to be encoded in the pointer itself. + */ +uint32_t ht_jhash(void *key, uint32_t length, uint32_t initval) +{ + uint32_t ret; + void *vkey; + + if (length <= sizeof(void *)) + vkey = &key; + else + vkey = key; + ret = jhash(vkey, length, initval); + return ret; +} diff --git a/tests/Makefile.am b/tests/Makefile.am index 674260f..73dce45 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -15,7 +15,7 @@ noinst_PROGRAMS = test_urcu test_urcu_dynamic_link test_urcu_timing \ test_urcu_bp test_urcu_bp_dynamic_link test_cycles_per_loop \ test_urcu_lfq test_urcu_wfq test_urcu_lfs test_urcu_wfs \ test_urcu_wfq_dynlink test_urcu_wfs_dynlink \ - test_urcu_lfq_dynlink test_urcu_lfs_dynlink + test_urcu_lfq_dynlink test_urcu_lfs_dynlink test_urcu_hash noinst_HEADERS = rcutorture.h if COMPAT_ARCH @@ -175,6 +175,9 @@ test_urcu_wfs_dynlink_SOURCES = test_urcu_wfs.c test_urcu_wfs_dynlink_CFLAGS = -DDYNAMIC_LINK_TEST $(AM_CFLAGS) test_urcu_wfs_dynlink_LDADD = $(URCU_CDS_LIB) +test_urcu_hash_SOURCES = test_urcu_hash.c $(COMPAT) +test_ht_LDADD = $(URCU_CDS_LIB) + urcutorture.c: api.h check-am: diff --git a/tests/test_ht.c b/tests/test_ht.c deleted file mode 100644 index cdc78d5..0000000 --- a/tests/test_ht.c +++ /dev/null @@ -1,456 +0,0 @@ -/* - * test_ht.c - * - * Userspace RCU library - test program - * - * Copyright February 2009 - Mathieu Desnoyers - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "../arch.h" - -#define HASH_SIZE 32 -#define RAND_POOL 1000 - -/* Make this big enough to include the POWER5+ L3 cacheline size of 256B */ -#define CACHE_LINE_SIZE 4096 - -/* hardcoded number of CPUs */ -#define NR_CPUS 16384 - -#if defined(_syscall0) -_syscall0(pid_t, gettid) -#elif defined(__NR_gettid) -static inline pid_t gettid(void) -{ - return syscall(__NR_gettid); -} -#else -#warning "use pid as tid" -static inline pid_t gettid(void) -{ - return getpid(); -} -#endif - -#ifndef DYNAMIC_LINK_TEST -#define _LGPL_SOURCE -#else -#define debug_yield_read() -#endif -#include "../urcu.h" - -static unsigned int __thread rand_lookup; -static unsigned long __thread nr_add; -static unsigned long __thread nr_addexist; -static unsigned long __thread nr_del; -static unsigned long __thread nr_delnoent; -static unsigned long __thread lookup_fail; -static unsigned long __thread lookup_ok; - -static struct rcu_ht *test_ht; - -struct test_data { - int a; - int b; -}; - -static volatile int test_go, test_stop; - -static unsigned long wdelay; - -static unsigned long duration; - -/* read-side C.S. duration, in loops */ -static unsigned long rduration; - -static inline void loop_sleep(unsigned long l) -{ - while(l-- != 0) - cpu_relax(); -} - -static int verbose_mode; - -#define printf_verbose(fmt, args...) \ - do { \ - if (verbose_mode) \ - printf(fmt, args); \ - } while (0) - -static unsigned int cpu_affinities[NR_CPUS]; -static unsigned int next_aff = 0; -static int use_affinity = 0; - -pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; - -static void set_affinity(void) -{ - cpu_set_t mask; - int cpu; - int ret; - - if (!use_affinity) - return; - - ret = pthread_mutex_lock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } - cpu = cpu_affinities[next_aff++]; - ret = pthread_mutex_unlock(&affinity_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } - CPU_ZERO(&mask); - CPU_SET(cpu, &mask); - sched_setaffinity(0, sizeof(mask), &mask); -} - -/* - * returns 0 if test should end. - */ -static int test_duration_write(void) -{ - return !test_stop; -} - -static int test_duration_read(void) -{ - return !test_stop; -} - -static unsigned long long __thread nr_writes; -static unsigned long long __thread nr_reads; - -static unsigned int nr_readers; -static unsigned int nr_writers; - -pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; - -void rcu_copy_mutex_lock(void) -{ - int ret; - ret = pthread_mutex_lock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex lock"); - exit(-1); - } -} - -void rcu_copy_mutex_unlock(void) -{ - int ret; - - ret = pthread_mutex_unlock(&rcu_copy_mutex); - if (ret) { - perror("Error in pthread mutex unlock"); - exit(-1); - } -} - -#define ARRAY_POISON 0xDEADBEEF - -void *thr_reader(void *_count) -{ - unsigned long long *count = _count; - struct test_data *local_ptr; - - printf_verbose("thread_begin %s, thread id : %lx, tid %lu\n", - "reader", pthread_self(), (unsigned long)gettid()); - - set_affinity(); - - rcu_register_thread(); - - while (!test_go) - { - } - smp_mb(); - - for (;;) { - rcu_read_lock(); - local_ptr = ht_lookup(test_ht, - (void *)(unsigned long)(rand_r(&rand_lookup) % RAND_POOL)); - if (local_ptr == NULL) - lookup_fail++; - else - lookup_ok++; - debug_yield_read(); - if (unlikely(rduration)) - loop_sleep(rduration); - rcu_read_unlock(); - nr_reads++; - if (unlikely(!test_duration_read())) - break; - } - - rcu_unregister_thread(); - - *count = nr_reads; - printf_verbose("thread_end %s, thread id : %lx, tid %lu\n", - "reader", pthread_self(), (unsigned long)gettid()); - printf_verbose("readid : %lx, lookupfail %lu, lookupok %lu\n", - pthread_self(), lookup_fail, lookup_ok); - return ((void*)1); - -} - -void *thr_writer(void *_count) -{ - unsigned long long *count = _count; - struct test_data *data; - int ret; - - printf_verbose("thread_begin %s, thread id : %lx, tid %lu\n", - "writer", pthread_self(), (unsigned long)gettid()); - - set_affinity(); - - rcu_register_thread(); - rcu_defer_register_thread(); - - while (!test_go) - { - } - smp_mb(); - - for (;;) { - if (rand_r(&rand_lookup) & 1) { - data = malloc(sizeof(struct test_data)); - //rcu_copy_mutex_lock(); - ret = ht_add(test_ht, - (void *)(unsigned long)(rand_r(&rand_lookup) % RAND_POOL), - data); - if (ret == -EEXIST) { - free(data); - nr_addexist++; - } else { - nr_add++; - } - //rcu_copy_mutex_unlock(); - } else { - /* May delete */ - //rcu_copy_mutex_lock(); - ret = ht_delete(test_ht, - (void *)(unsigned long)(rand_r(&rand_lookup) % RAND_POOL)); - if (ret == -ENOENT) - nr_delnoent++; - else - nr_del++; - //rcu_copy_mutex_unlock(); - } - //if (nr_writes % 100000 == 0) { - if (nr_writes % 1000 == 0) { - if (rand_r(&rand_lookup) & 1) { - ht_resize(test_ht, 1); - } else { - ht_resize(test_ht, -1); - } - } - nr_writes++; - if (unlikely(!test_duration_write())) - break; - if (unlikely(wdelay)) - loop_sleep(wdelay); - } - - rcu_defer_unregister_thread(); - rcu_unregister_thread(); - - printf_verbose("thread_end %s, thread id : %lx, tid %lu\n", - "writer", pthread_self(), (unsigned long)gettid()); - printf_verbose("info id %lx: nr_add %lu, nr_addexist %lu, nr_del %lu, " - "nr_delnoent %lu\n", pthread_self(), nr_add, - nr_addexist, nr_del, nr_delnoent); - *count = nr_writes; - return ((void*)2); -} - -void show_usage(int argc, char **argv) -{ - printf("Usage : %s nr_readers nr_writers duration (s)", argv[0]); -#ifdef DEBUG_YIELD - printf(" [-r] [-w] (yield reader and/or writer)"); -#endif - printf(" [-d delay] (writer period (us))"); - printf(" [-c duration] (reader C.S. duration (in loops))"); - printf(" [-v] (verbose output)"); - printf(" [-a cpu#] [-a cpu#]... (affinity)"); - printf("\n"); -} - -int main(int argc, char **argv) -{ - int err; - pthread_t *tid_reader, *tid_writer; - void *tret; - unsigned long long *count_reader, *count_writer; - unsigned long long tot_reads = 0, tot_writes = 0; - int i, a, ret; - - if (argc < 4) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[1], "%u", &nr_readers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[2], "%u", &nr_writers); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - err = sscanf(argv[3], "%lu", &duration); - if (err != 1) { - show_usage(argc, argv); - return -1; - } - - for (i = 4; i < argc; i++) { - if (argv[i][0] != '-') - continue; - switch (argv[i][1]) { -#ifdef DEBUG_YIELD - case 'r': - yield_active |= YIELD_READ; - break; - case 'w': - yield_active |= YIELD_WRITE; - break; -#endif - case 'a': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - a = atoi(argv[++i]); - cpu_affinities[next_aff++] = a; - use_affinity = 1; - printf_verbose("Adding CPU %d affinity\n", a); - break; - case 'c': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - rduration = atol(argv[++i]); - break; - case 'd': - if (argc < i + 2) { - show_usage(argc, argv); - return -1; - } - wdelay = atol(argv[++i]); - break; - case 'v': - verbose_mode = 1; - break; - } - } - - printf_verbose("running test for %lu seconds, %u readers, %u writers.\n", - duration, nr_readers, nr_writers); - printf_verbose("Writer delay : %lu loops.\n", wdelay); - printf_verbose("Reader duration : %lu loops.\n", rduration); - printf_verbose("thread %-6s, thread id : %lx, tid %lu\n", - "main", pthread_self(), (unsigned long)gettid()); - - tid_reader = malloc(sizeof(*tid_reader) * nr_readers); - tid_writer = malloc(sizeof(*tid_writer) * nr_writers); - count_reader = malloc(sizeof(*count_reader) * nr_readers); - count_writer = malloc(sizeof(*count_writer) * nr_writers); - test_ht = ht_new(ht_jhash, free, HASH_SIZE, sizeof(unsigned long), - 43223455); - next_aff = 0; - - for (i = 0; i < nr_readers; i++) { - err = pthread_create(&tid_reader[i], NULL, thr_reader, - &count_reader[i]); - if (err != 0) - exit(1); - } - for (i = 0; i < nr_writers; i++) { - err = pthread_create(&tid_writer[i], NULL, thr_writer, - &count_writer[i]); - if (err != 0) - exit(1); - } - - smp_mb(); - - test_go = 1; - - sleep(duration); - - test_stop = 1; - - for (i = 0; i < nr_readers; i++) { - err = pthread_join(tid_reader[i], &tret); - if (err != 0) - exit(1); - tot_reads += count_reader[i]; - } - for (i = 0; i < nr_writers; i++) { - err = pthread_join(tid_writer[i], &tret); - if (err != 0) - exit(1); - tot_writes += count_writer[i]; - } - rcu_register_thread(); - rcu_defer_register_thread(); - ret = ht_destroy(test_ht); - rcu_defer_unregister_thread(); - rcu_unregister_thread(); - - printf_verbose("final delete: %d items\n", ret); - printf_verbose("total number of reads : %llu, writes %llu\n", tot_reads, - tot_writes); - printf("SUMMARY %-25s testdur %4lu nr_readers %3u rdur %6lu " - "nr_writers %3u " - "wdelay %6lu nr_reads %12llu nr_writes %12llu nr_ops %12llu\n", - argv[0], duration, nr_readers, rduration, - nr_writers, wdelay, tot_reads, tot_writes, - tot_reads + tot_writes); - free(tid_reader); - free(tid_writer); - free(count_reader); - free(count_writer); - return 0; -} diff --git a/tests/test_urcu_hash.c b/tests/test_urcu_hash.c new file mode 100644 index 0000000..cdc78d5 --- /dev/null +++ b/tests/test_urcu_hash.c @@ -0,0 +1,456 @@ +/* + * test_ht.c + * + * Userspace RCU library - test program + * + * Copyright February 2009 - Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../arch.h" + +#define HASH_SIZE 32 +#define RAND_POOL 1000 + +/* Make this big enough to include the POWER5+ L3 cacheline size of 256B */ +#define CACHE_LINE_SIZE 4096 + +/* hardcoded number of CPUs */ +#define NR_CPUS 16384 + +#if defined(_syscall0) +_syscall0(pid_t, gettid) +#elif defined(__NR_gettid) +static inline pid_t gettid(void) +{ + return syscall(__NR_gettid); +} +#else +#warning "use pid as tid" +static inline pid_t gettid(void) +{ + return getpid(); +} +#endif + +#ifndef DYNAMIC_LINK_TEST +#define _LGPL_SOURCE +#else +#define debug_yield_read() +#endif +#include "../urcu.h" + +static unsigned int __thread rand_lookup; +static unsigned long __thread nr_add; +static unsigned long __thread nr_addexist; +static unsigned long __thread nr_del; +static unsigned long __thread nr_delnoent; +static unsigned long __thread lookup_fail; +static unsigned long __thread lookup_ok; + +static struct rcu_ht *test_ht; + +struct test_data { + int a; + int b; +}; + +static volatile int test_go, test_stop; + +static unsigned long wdelay; + +static unsigned long duration; + +/* read-side C.S. duration, in loops */ +static unsigned long rduration; + +static inline void loop_sleep(unsigned long l) +{ + while(l-- != 0) + cpu_relax(); +} + +static int verbose_mode; + +#define printf_verbose(fmt, args...) \ + do { \ + if (verbose_mode) \ + printf(fmt, args); \ + } while (0) + +static unsigned int cpu_affinities[NR_CPUS]; +static unsigned int next_aff = 0; +static int use_affinity = 0; + +pthread_mutex_t affinity_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void set_affinity(void) +{ + cpu_set_t mask; + int cpu; + int ret; + + if (!use_affinity) + return; + + ret = pthread_mutex_lock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } + cpu = cpu_affinities[next_aff++]; + ret = pthread_mutex_unlock(&affinity_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); + sched_setaffinity(0, sizeof(mask), &mask); +} + +/* + * returns 0 if test should end. + */ +static int test_duration_write(void) +{ + return !test_stop; +} + +static int test_duration_read(void) +{ + return !test_stop; +} + +static unsigned long long __thread nr_writes; +static unsigned long long __thread nr_reads; + +static unsigned int nr_readers; +static unsigned int nr_writers; + +pthread_mutex_t rcu_copy_mutex = PTHREAD_MUTEX_INITIALIZER; + +void rcu_copy_mutex_lock(void) +{ + int ret; + ret = pthread_mutex_lock(&rcu_copy_mutex); + if (ret) { + perror("Error in pthread mutex lock"); + exit(-1); + } +} + +void rcu_copy_mutex_unlock(void) +{ + int ret; + + ret = pthread_mutex_unlock(&rcu_copy_mutex); + if (ret) { + perror("Error in pthread mutex unlock"); + exit(-1); + } +} + +#define ARRAY_POISON 0xDEADBEEF + +void *thr_reader(void *_count) +{ + unsigned long long *count = _count; + struct test_data *local_ptr; + + printf_verbose("thread_begin %s, thread id : %lx, tid %lu\n", + "reader", pthread_self(), (unsigned long)gettid()); + + set_affinity(); + + rcu_register_thread(); + + while (!test_go) + { + } + smp_mb(); + + for (;;) { + rcu_read_lock(); + local_ptr = ht_lookup(test_ht, + (void *)(unsigned long)(rand_r(&rand_lookup) % RAND_POOL)); + if (local_ptr == NULL) + lookup_fail++; + else + lookup_ok++; + debug_yield_read(); + if (unlikely(rduration)) + loop_sleep(rduration); + rcu_read_unlock(); + nr_reads++; + if (unlikely(!test_duration_read())) + break; + } + + rcu_unregister_thread(); + + *count = nr_reads; + printf_verbose("thread_end %s, thread id : %lx, tid %lu\n", + "reader", pthread_self(), (unsigned long)gettid()); + printf_verbose("readid : %lx, lookupfail %lu, lookupok %lu\n", + pthread_self(), lookup_fail, lookup_ok); + return ((void*)1); + +} + +void *thr_writer(void *_count) +{ + unsigned long long *count = _count; + struct test_data *data; + int ret; + + printf_verbose("thread_begin %s, thread id : %lx, tid %lu\n", + "writer", pthread_self(), (unsigned long)gettid()); + + set_affinity(); + + rcu_register_thread(); + rcu_defer_register_thread(); + + while (!test_go) + { + } + smp_mb(); + + for (;;) { + if (rand_r(&rand_lookup) & 1) { + data = malloc(sizeof(struct test_data)); + //rcu_copy_mutex_lock(); + ret = ht_add(test_ht, + (void *)(unsigned long)(rand_r(&rand_lookup) % RAND_POOL), + data); + if (ret == -EEXIST) { + free(data); + nr_addexist++; + } else { + nr_add++; + } + //rcu_copy_mutex_unlock(); + } else { + /* May delete */ + //rcu_copy_mutex_lock(); + ret = ht_delete(test_ht, + (void *)(unsigned long)(rand_r(&rand_lookup) % RAND_POOL)); + if (ret == -ENOENT) + nr_delnoent++; + else + nr_del++; + //rcu_copy_mutex_unlock(); + } + //if (nr_writes % 100000 == 0) { + if (nr_writes % 1000 == 0) { + if (rand_r(&rand_lookup) & 1) { + ht_resize(test_ht, 1); + } else { + ht_resize(test_ht, -1); + } + } + nr_writes++; + if (unlikely(!test_duration_write())) + break; + if (unlikely(wdelay)) + loop_sleep(wdelay); + } + + rcu_defer_unregister_thread(); + rcu_unregister_thread(); + + printf_verbose("thread_end %s, thread id : %lx, tid %lu\n", + "writer", pthread_self(), (unsigned long)gettid()); + printf_verbose("info id %lx: nr_add %lu, nr_addexist %lu, nr_del %lu, " + "nr_delnoent %lu\n", pthread_self(), nr_add, + nr_addexist, nr_del, nr_delnoent); + *count = nr_writes; + return ((void*)2); +} + +void show_usage(int argc, char **argv) +{ + printf("Usage : %s nr_readers nr_writers duration (s)", argv[0]); +#ifdef DEBUG_YIELD + printf(" [-r] [-w] (yield reader and/or writer)"); +#endif + printf(" [-d delay] (writer period (us))"); + printf(" [-c duration] (reader C.S. duration (in loops))"); + printf(" [-v] (verbose output)"); + printf(" [-a cpu#] [-a cpu#]... (affinity)"); + printf("\n"); +} + +int main(int argc, char **argv) +{ + int err; + pthread_t *tid_reader, *tid_writer; + void *tret; + unsigned long long *count_reader, *count_writer; + unsigned long long tot_reads = 0, tot_writes = 0; + int i, a, ret; + + if (argc < 4) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[1], "%u", &nr_readers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[2], "%u", &nr_writers); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + err = sscanf(argv[3], "%lu", &duration); + if (err != 1) { + show_usage(argc, argv); + return -1; + } + + for (i = 4; i < argc; i++) { + if (argv[i][0] != '-') + continue; + switch (argv[i][1]) { +#ifdef DEBUG_YIELD + case 'r': + yield_active |= YIELD_READ; + break; + case 'w': + yield_active |= YIELD_WRITE; + break; +#endif + case 'a': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + a = atoi(argv[++i]); + cpu_affinities[next_aff++] = a; + use_affinity = 1; + printf_verbose("Adding CPU %d affinity\n", a); + break; + case 'c': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + rduration = atol(argv[++i]); + break; + case 'd': + if (argc < i + 2) { + show_usage(argc, argv); + return -1; + } + wdelay = atol(argv[++i]); + break; + case 'v': + verbose_mode = 1; + break; + } + } + + printf_verbose("running test for %lu seconds, %u readers, %u writers.\n", + duration, nr_readers, nr_writers); + printf_verbose("Writer delay : %lu loops.\n", wdelay); + printf_verbose("Reader duration : %lu loops.\n", rduration); + printf_verbose("thread %-6s, thread id : %lx, tid %lu\n", + "main", pthread_self(), (unsigned long)gettid()); + + tid_reader = malloc(sizeof(*tid_reader) * nr_readers); + tid_writer = malloc(sizeof(*tid_writer) * nr_writers); + count_reader = malloc(sizeof(*count_reader) * nr_readers); + count_writer = malloc(sizeof(*count_writer) * nr_writers); + test_ht = ht_new(ht_jhash, free, HASH_SIZE, sizeof(unsigned long), + 43223455); + next_aff = 0; + + for (i = 0; i < nr_readers; i++) { + err = pthread_create(&tid_reader[i], NULL, thr_reader, + &count_reader[i]); + if (err != 0) + exit(1); + } + for (i = 0; i < nr_writers; i++) { + err = pthread_create(&tid_writer[i], NULL, thr_writer, + &count_writer[i]); + if (err != 0) + exit(1); + } + + smp_mb(); + + test_go = 1; + + sleep(duration); + + test_stop = 1; + + for (i = 0; i < nr_readers; i++) { + err = pthread_join(tid_reader[i], &tret); + if (err != 0) + exit(1); + tot_reads += count_reader[i]; + } + for (i = 0; i < nr_writers; i++) { + err = pthread_join(tid_writer[i], &tret); + if (err != 0) + exit(1); + tot_writes += count_writer[i]; + } + rcu_register_thread(); + rcu_defer_register_thread(); + ret = ht_destroy(test_ht); + rcu_defer_unregister_thread(); + rcu_unregister_thread(); + + printf_verbose("final delete: %d items\n", ret); + printf_verbose("total number of reads : %llu, writes %llu\n", tot_reads, + tot_writes); + printf("SUMMARY %-25s testdur %4lu nr_readers %3u rdur %6lu " + "nr_writers %3u " + "wdelay %6lu nr_reads %12llu nr_writes %12llu nr_ops %12llu\n", + argv[0], duration, nr_readers, rduration, + nr_writers, wdelay, tot_reads, tot_writes, + tot_reads + tot_writes); + free(tid_reader); + free(tid_writer); + free(count_reader); + free(count_writer); + return 0; +} diff --git a/urcu-ht.c b/urcu-ht.c deleted file mode 100644 index b8777ac..0000000 --- a/urcu-ht.c +++ /dev/null @@ -1,485 +0,0 @@ - -/* - * TODO: keys are currently assumed <= sizeof(void *). Key target never freed. - */ - -#define _LGPL_SOURCE -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* - * Maximum number of hash table buckets: 256M on 64-bit. - * Should take about 512MB max if we assume 1 node per 4 buckets. - */ -#define MAX_HT_BUCKETS ((256 << 10) / sizeof(void *)) - -/* node flags */ -#define NODE_STOLEN (1 << 0) - -struct rcu_ht_node; - -struct rcu_ht_node { - struct rcu_ht_node *next; - void *key; - void *data; - unsigned int flags; -}; - -struct rcu_table { - unsigned long size; - struct rcu_ht_node *tbl[0]; -}; - -struct rcu_ht { - struct rcu_table *t; /* shared */ - ht_hash_fct hash_fct; - void (*free_fct)(void *data); /* fct to free data */ - uint32_t keylen; - uint32_t hashseed; - pthread_mutex_t resize_mutex; /* resize mutex: add/del mutex */ - int resize_ongoing; /* fast-path resize check */ -}; - -struct rcu_ht *ht_new(ht_hash_fct hash_fct, void (*free_fct)(void *data), - unsigned long init_size, uint32_t keylen, - uint32_t hashseed) -{ - struct rcu_ht *ht; - - ht = calloc(1, sizeof(struct rcu_ht)); - ht->hash_fct = hash_fct; - ht->free_fct = free_fct; - ht->keylen = keylen; - ht->hashseed = hashseed; - /* this mutex should not nest in read-side C.S. */ - pthread_mutex_init(&ht->resize_mutex, NULL); - ht->resize_ongoing = 0; /* shared */ - ht->t = calloc(1, sizeof(struct rcu_table) - + (init_size * sizeof(struct rcu_ht_node *))); - ht->t->size = init_size; - return ht; -} - -void *ht_lookup(struct rcu_ht *ht, void *key) -{ - struct rcu_table *t; - unsigned long hash; - struct rcu_ht_node *node; - void *ret; - - rcu_read_lock(); - t = rcu_dereference(ht->t); - smp_read_barrier_depends(); /* read t before size and table */ - hash = ht->hash_fct(key, ht->keylen, ht->hashseed) % t->size; - smp_read_barrier_depends(); /* read size before links */ - node = rcu_dereference(t->tbl[hash]); - for (;;) { - if (likely(!node)) { - ret = NULL; - break; - } - if (node->key == key) { - ret = node->data; - break; - } - node = rcu_dereference(node->next); - } - rcu_read_unlock(); - - return ret; -} - -/* - * Will re-try until either: - * - The key is already there (-EEXIST) - * - We successfully add the key at the head of a table bucket. - */ -int ht_add(struct rcu_ht *ht, void *key, void *data) -{ - struct rcu_ht_node *node, *old_head, *new_head; - struct rcu_table *t; - unsigned long hash; - int ret = 0; - - new_head = calloc(1, sizeof(struct rcu_ht_node)); - new_head->key = key; - new_head->data = data; - new_head->flags = 0; - /* here comes the fun and tricky part. - * Add at the beginning with a cmpxchg. - * Hold a read lock between the moment the first element is read - * and the nodes traversal (to find duplicates). This ensures - * the head pointer has not been reclaimed when cmpxchg is done. - * Always adding at the head ensures that we would have to - * re-try if a new item has been added concurrently. So we ensure that - * we never add duplicates. */ -retry: - rcu_read_lock(); - - if (unlikely(LOAD_SHARED(ht->resize_ongoing))) { - rcu_read_unlock(); - /* - * Wait for resize to complete before continuing. - */ - ret = pthread_mutex_lock(&ht->resize_mutex); - assert(!ret); - ret = pthread_mutex_unlock(&ht->resize_mutex); - assert(!ret); - goto retry; - } - - t = rcu_dereference(ht->t); - /* no read barrier needed, because no concurrency with resize */ - hash = ht->hash_fct(key, ht->keylen, ht->hashseed) % t->size; - - old_head = node = rcu_dereference(t->tbl[hash]); - for (;;) { - if (likely(!node)) { - break; - } - if (node->key == key) { - ret = -EEXIST; - goto end; - } - node = rcu_dereference(node->next); - } - new_head->next = old_head; - if (rcu_cmpxchg_pointer(&t->tbl[hash], old_head, new_head) != old_head) - goto restart; -end: - rcu_read_unlock(); - return ret; - - /* restart loop, release and re-take the read lock to be kind to GP */ -restart: - rcu_read_unlock(); - goto retry; -} - -/* - * Restart until we successfully remove the entry, or no entry is left - * ((void *)(unsigned long)-ENOENT). - * Deal with concurrent stealers by doing an extra verification pass to check - * that no element in the list are still pointing to the element stolen. - * This could happen if two concurrent steal for consecutive objects are - * executed. A pointer to an object being stolen could be saved by the - * concurrent stealer for the previous object. - * Also, given that in this precise scenario, another stealer can also want to - * delete the doubly-referenced object; use a "stolen" flag to let only one - * stealer delete the object. - */ -void *ht_steal(struct rcu_ht *ht, void *key) -{ - struct rcu_ht_node **prev, *node, *del_node = NULL; - struct rcu_table *t; - unsigned long hash; - void *data; - int ret; - -retry: - rcu_read_lock(); - - if (unlikely(LOAD_SHARED(ht->resize_ongoing))) { - rcu_read_unlock(); - /* - * Wait for resize to complete before continuing. - */ - ret = pthread_mutex_lock(&ht->resize_mutex); - assert(!ret); - ret = pthread_mutex_unlock(&ht->resize_mutex); - assert(!ret); - goto retry; - } - - t = rcu_dereference(ht->t); - /* no read barrier needed, because no concurrency with resize */ - hash = ht->hash_fct(key, ht->keylen, ht->hashseed) % t->size; - - prev = &t->tbl[hash]; - node = rcu_dereference(*prev); - for (;;) { - if (likely(!node)) { - if (del_node) { - goto end; - } else { - goto error; - } - } - if (node->key == key) { - break; - } - prev = &node->next; - node = rcu_dereference(*prev); - } - - if (!del_node) { - /* - * Another concurrent thread stole it ? If so, let it deal with - * this. Assume NODE_STOLEN is the only flag. If this changes, - * read flags before cmpxchg. - */ - if (cmpxchg(&node->flags, 0, NODE_STOLEN) != 0) - goto error; - } - - /* Found it ! pointer to object is in "prev" */ - if (rcu_cmpxchg_pointer(prev, node, node->next) == node) - del_node = node; - goto restart; - -end: - /* - * From that point, we own node. Note that there can still be concurrent - * RCU readers using it. We can free it outside of read lock after a GP. - */ - rcu_read_unlock(); - - data = del_node->data; - call_rcu(free, del_node); - return data; - -error: - data = (void *)(unsigned long)-ENOENT; - rcu_read_unlock(); - return data; - - /* restart loop, release and re-take the read lock to be kind to GP */ -restart: - rcu_read_unlock(); - goto retry; -} - -int ht_delete(struct rcu_ht *ht, void *key) -{ - void *data; - - data = ht_steal(ht, key); - if (data && data != (void *)(unsigned long)-ENOENT) { - if (ht->free_fct) - call_rcu(ht->free_fct, data); - return 0; - } else { - return -ENOENT; - } -} - -/* Delete all old elements. Allow concurrent writer accesses. */ -int ht_delete_all(struct rcu_ht *ht) -{ - unsigned long i; - struct rcu_ht_node **prev, *node, *inext; - struct rcu_table *t; - int cnt = 0; - int ret; - - /* - * Mutual exclusion with resize operations, but leave add/steal execute - * concurrently. This is OK because we operate only on the heads. - */ - ret = pthread_mutex_lock(&ht->resize_mutex); - assert(!ret); - - t = rcu_dereference(ht->t); - /* no read barrier needed, because no concurrency with resize */ - for (i = 0; i < t->size; i++) { - rcu_read_lock(); - prev = &t->tbl[i]; - /* - * Cut the head. After that, we own the first element. - */ - node = rcu_xchg_pointer(prev, NULL); - if (!node) { - rcu_read_unlock(); - continue; - } - /* - * We manage a list shared with concurrent writers and readers. - * Note that a concurrent add may or may not be deleted by us, - * depending if it arrives before or after the head is cut. - * "node" points to our first node. Remove first elements - * iteratively. - */ - for (;;) { - inext = NULL; - prev = &node->next; - if (prev) - inext = rcu_xchg_pointer(prev, NULL); - /* - * "node" is the first element of the list we have cut. - * We therefore own it, no concurrent writer may delete - * it. There can only be concurrent lookups. Concurrent - * add can only be done on a bucket head, but we've cut - * it already. inext is also owned by us, because we - * have exchanged it for "NULL". It will therefore be - * safe to use it after a G.P. - */ - rcu_read_unlock(); - if (node->data) - call_rcu(ht->free_fct, node->data); - call_rcu(free, node); - cnt++; - if (likely(!inext)) - break; - rcu_read_lock(); - node = inext; - } - } - - ret = pthread_mutex_unlock(&ht->resize_mutex); - assert(!ret); - return cnt; -} - -/* - * Should only be called when no more concurrent readers nor writers can - * possibly access the table. - */ -int ht_destroy(struct rcu_ht *ht) -{ - int ret; - - ret = ht_delete_all(ht); - free(ht->t); - free(ht); - return ret; -} - -static void ht_resize_grow(struct rcu_ht *ht) -{ - unsigned long i, new_size, old_size; - struct rcu_table *new_t, *old_t; - struct rcu_ht_node *node, *new_node, *tmp; - unsigned long hash; - - old_t = ht->t; - old_size = old_t->size; - - if (old_size == MAX_HT_BUCKETS) - return; - - new_size = old_size << 1; - new_t = calloc(1, sizeof(struct rcu_table) - + (new_size * sizeof(struct rcu_ht_node *))); - new_t->size = new_size; - - for (i = 0; i < old_size; i++) { - /* - * Re-hash each entry, insert in new table. - * It's important that a reader looking for a key _will_ find it - * if it's in the table. - * Copy each node. (just the node, not ->data) - */ - node = old_t->tbl[i]; - while (node) { - hash = ht->hash_fct(node->key, ht->keylen, ht->hashseed) - % new_size; - new_node = malloc(sizeof(struct rcu_ht_node)); - new_node->key = node->key; - new_node->data = node->data; - new_node->flags = node->flags; - new_node->next = new_t->tbl[hash]; /* link to first */ - new_t->tbl[hash] = new_node; /* add to head */ - node = node->next; - } - } - - /* Changing table and size atomically wrt lookups */ - rcu_assign_pointer(ht->t, new_t); - - /* Ensure all concurrent lookups use new size and table */ - synchronize_rcu(); - - for (i = 0; i < old_size; i++) { - node = old_t->tbl[i]; - while (node) { - tmp = node->next; - free(node); - node = tmp; - } - } - free(old_t); -} - -static void ht_resize_shrink(struct rcu_ht *ht) -{ - unsigned long i, new_size; - struct rcu_table *new_t, *old_t; - struct rcu_ht_node **prev, *node; - - old_t = ht->t; - if (old_t->size == 1) - return; - - new_size = old_t->size >> 1; - - for (i = 0; i < new_size; i++) { - /* Link end with first entry of i + new_size */ - prev = &old_t->tbl[i]; - node = *prev; - while (node) { - prev = &node->next; - node = *prev; - } - *prev = old_t->tbl[i + new_size]; - } - smp_wmb(); /* write links before changing size */ - STORE_SHARED(old_t->size, new_size); - - /* Ensure all concurrent lookups use new size */ - synchronize_rcu(); - - new_t = realloc(old_t, sizeof(struct rcu_table) - + (new_size * sizeof(struct rcu_ht_node *))); - /* shrinking, pointers should not move */ - assert(new_t == old_t); -} - -/* - * growth: >0: *2, <0: /2 - */ -void ht_resize(struct rcu_ht *ht, int growth) -{ - int ret; - - ret = pthread_mutex_lock(&ht->resize_mutex); - assert(!ret); - STORE_SHARED(ht->resize_ongoing, 1); - synchronize_rcu(); - /* All add/remove are waiting on the mutex. */ - if (growth > 0) - ht_resize_grow(ht); - else if (growth < 0) - ht_resize_shrink(ht); - smp_mb(); - STORE_SHARED(ht->resize_ongoing, 0); - ret = pthread_mutex_unlock(&ht->resize_mutex); - assert(!ret); -} - -/* - * Expects keys <= than pointer size to be encoded in the pointer itself. - */ -uint32_t ht_jhash(void *key, uint32_t length, uint32_t initval) -{ - uint32_t ret; - void *vkey; - - if (length <= sizeof(void *)) - vkey = &key; - else - vkey = key; - ret = jhash(vkey, length, initval); - return ret; -} diff --git a/urcu-ht.h b/urcu-ht.h deleted file mode 100644 index 7e3c36c..0000000 --- a/urcu-ht.h +++ /dev/null @@ -1,38 +0,0 @@ -#ifndef _URCU_HT_H -#define _URCU_HT_H - -#include - -/* - * Caution ! - * Ensure writer threads are registered as urcu readers and with with - * urcu-defer. - * Ensure reader threads are registered as urcu readers. - */ - -typedef uint32_t (*ht_hash_fct)(void *key, uint32_t length, uint32_t initval); - -/* - * init_size must be power of two. - */ -struct rcu_ht *ht_new(ht_hash_fct hash_fct, void (*free_fct)(void *data), - unsigned long init_size, uint32_t keylen, - uint32_t hashseed); - -int ht_delete_all(struct rcu_ht *ht); - -int ht_destroy(struct rcu_ht *ht); - -void *ht_lookup(struct rcu_ht *ht, void *key); - -int ht_add(struct rcu_ht *ht, void *key, void *data); - -int ht_delete(struct rcu_ht *ht, void *key); - -void *ht_steal(struct rcu_ht *ht, void *key); - -void ht_resize(struct rcu_ht *ht, int growth); - -uint32_t ht_jhash(void *key, uint32_t length, uint32_t initval); - -#endif /* _URCU_HT_H */ diff --git a/urcu/cds.h b/urcu/cds.h index f37a63a..ea74cf1 100644 --- a/urcu/cds.h +++ b/urcu/cds.h @@ -29,6 +29,7 @@ #include #include #include +#include #include #include diff --git a/urcu/rculfhash.h b/urcu/rculfhash.h new file mode 100644 index 0000000..d989221 --- /dev/null +++ b/urcu/rculfhash.h @@ -0,0 +1,38 @@ +#ifndef _URCU_RCULFHASH_H +#define _URCU_RCULFHASH_H + +#include + +/* + * Caution ! + * Ensure writer threads are registered as urcu readers and with with + * urcu-defer. + * Ensure reader threads are registered as urcu readers. + */ + +typedef uint32_t (*ht_hash_fct)(void *key, uint32_t length, uint32_t initval); + +/* + * init_size must be power of two. + */ +struct rcu_ht *ht_new(ht_hash_fct hash_fct, void (*free_fct)(void *data), + unsigned long init_size, uint32_t keylen, + uint32_t hashseed); + +int ht_delete_all(struct rcu_ht *ht); + +int ht_destroy(struct rcu_ht *ht); + +void *ht_lookup(struct rcu_ht *ht, void *key); + +int ht_add(struct rcu_ht *ht, void *key, void *data); + +int ht_delete(struct rcu_ht *ht, void *key); + +void *ht_steal(struct rcu_ht *ht, void *key); + +void ht_resize(struct rcu_ht *ht, int growth); + +uint32_t ht_jhash(void *key, uint32_t length, uint32_t initval); + +#endif /* _URCU_RCULFHASH_H */