src/common/hashtable/rculfhash-mm-mmap.c

   1 /*
   2  * rculfhash-mm-mmap.c
   3  *
   4  * mmap/reservation based memory management for Lock-Free RCU Hash Table
   5  *
   6  * Copyright 2011 - Lai Jiangshan <laijs@cn.fujitsu.com>
   7  *
   8  * This library is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License as published by the Free Software Foundation; either
  11  * version 2.1 of the License, or (at your option) any later version.
  12  *
  13  * This library is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16  * Lesser General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU Lesser General Public
  19  * License along with this library; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  21  */
  22
  23 #define _GNU_SOURCE
  24 #define _LGPL_SOURCE
  25 #include <unistd.h>
  26 #include <sys/mman.h>
  27 #include "rculfhash-internal.h"
  28
  29 #ifndef MAP_ANONYMOUS
  30 #define MAP_ANONYMOUS MAP_ANON
  31 #endif
  32
  33 /* reserve inaccessible memory space without allocation any memory */
  34 static void *memory_map(size_t length)
  35 {
  36         void *ret = mmap(NULL, length, PROT_NONE,
  37                         MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
  38
  39         assert(ret != MAP_FAILED);
  40         return ret;
  41 }
  42
  43 static void memory_unmap(void *ptr, size_t length)
  44 {
  45         int ret __attribute__((unused));
  46
  47         ret = munmap(ptr, length);
  48
  49         assert(ret == 0);
  50 }
  51
  52 static void memory_populate(void *ptr, size_t length)
  53 {
  54         void *ret __attribute__((unused));
  55
  56         ret = mmap(ptr, length, PROT_READ | PROT_WRITE,
  57                         MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
  58
  59         assert(ret == ptr);
  60 }
  61
  62 /*
  63  * Discard garbage memory and avoid system save it when try to swap it out.
  64  * Make it still reserved, inaccessible.
  65  */
  66 static void memory_discard(void *ptr, size_t length)
  67 {
  68         void *ret __attribute__((unused));
  69
  70         ret = mmap(ptr, length, PROT_NONE,
  71                         MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
  72
  73         assert(ret == ptr);
  74 }
  75
  76 static
  77 void cds_lfht_alloc_bucket_table(struct cds_lfht *ht, unsigned long order)
  78 {
  79         if (order == 0) {
  80                 if (ht->min_nr_alloc_buckets == ht->max_nr_buckets) {
  81                         /* small table */
  82                         ht->tbl_mmap = calloc(ht->max_nr_buckets,
  83                                         sizeof(*ht->tbl_mmap));
  84                         assert(ht->tbl_mmap);
  85                         return;
  86                 }
  87                 /* large table */
  88                 ht->tbl_mmap = memory_map(ht->max_nr_buckets
  89                         * sizeof(*ht->tbl_mmap));
  90                 memory_populate(ht->tbl_mmap,
  91                         ht->min_nr_alloc_buckets * sizeof(*ht->tbl_mmap));
  92         } else if (order > ht->min_alloc_buckets_order) {
  93                 /* large table */
  94                 unsigned long len = 1UL << (order - 1);
  95
  96                 assert(ht->min_nr_alloc_buckets < ht->max_nr_buckets);
  97                 memory_populate(ht->tbl_mmap + len,
  98                                 len * sizeof(*ht->tbl_mmap));
  99         }
 100         /* Nothing to do for 0 < order && order <= ht->min_alloc_buckets_order */
 101 }
 102
 103 /*
 104  * cds_lfht_free_bucket_table() should be called with decreasing order.
 105  * When cds_lfht_free_bucket_table(0) is called, it means the whole
 106  * lfht is destroyed.
 107  */
 108 static
 109 void cds_lfht_free_bucket_table(struct cds_lfht *ht, unsigned long order)
 110 {
 111         if (order == 0) {
 112                 if (ht->min_nr_alloc_buckets == ht->max_nr_buckets) {
 113                         /* small table */
 114                         poison_free(ht->tbl_mmap);
 115                         return;
 116                 }
 117                 /* large table */
 118                 memory_unmap(ht->tbl_mmap,
 119                         ht->max_nr_buckets * sizeof(*ht->tbl_mmap));
 120         } else if (order > ht->min_alloc_buckets_order) {
 121                 /* large table */
 122                 unsigned long len = 1UL << (order - 1);
 123
 124                 assert(ht->min_nr_alloc_buckets < ht->max_nr_buckets);
 125                 memory_discard(ht->tbl_mmap + len, len * sizeof(*ht->tbl_mmap));
 126         }
 127         /* Nothing to do for 0 < order && order <= ht->min_alloc_buckets_order */
 128 }
 129
 130 static
 131 struct cds_lfht_node *bucket_at(struct cds_lfht *ht, unsigned long index)
 132 {
 133         return &ht->tbl_mmap[index];
 134 }
 135
 136 static
 137 struct cds_lfht *alloc_cds_lfht(unsigned long min_nr_alloc_buckets,
 138                 unsigned long max_nr_buckets)
 139 {
 140         unsigned long page_bucket_size;
 141
 142         page_bucket_size = getpagesize() / sizeof(struct cds_lfht_node);
 143         if (max_nr_buckets <= page_bucket_size) {
 144                 /* small table */
 145                 min_nr_alloc_buckets = max_nr_buckets;
 146         } else {
 147                 /* large table */
 148                 min_nr_alloc_buckets = max(min_nr_alloc_buckets,
 149                                         page_bucket_size);
 150         }
 151
 152         return __default_alloc_cds_lfht(
 153                         &cds_lfht_mm_mmap, sizeof(struct cds_lfht),
 154                         min_nr_alloc_buckets, max_nr_buckets);
 155 }
 156
 157 const struct cds_lfht_mm_type cds_lfht_mm_mmap = {
 158         .alloc_cds_lfht = alloc_cds_lfht,
 159         .alloc_bucket_table = cds_lfht_alloc_bucket_table,
 160         .free_bucket_table = cds_lfht_free_bucket_table,
 161         .bucket_at = bucket_at,
 162 };