urcu/uatomic/generic.h

   1 #ifndef _URCU_UATOMIC_GENERIC_H
   2 #define _URCU_UATOMIC_GENERIC_H
   3
   4 /*
   5  * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
   6  * Copyright (c) 1996-1999 by Silicon Graphics.  All rights reserved.
   7  * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P.
   8  * Copyright (c) 2009      Mathieu Desnoyers
   9  * Copyright (c) 2010      Paolo Bonzini
  10  *
  11  * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
  12  * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
  13  *
  14  * Permission is hereby granted to use or copy this program
  15  * for any purpose,  provided the above notices are retained on all copies.
  16  * Permission to modify the code and to distribute modified code is granted,
  17  * provided the above notices are retained, and a notice that the code was
  18  * modified is included with the above copyright notice.
  19  *
  20  * Code inspired from libuatomic_ops-1.2, inherited in part from the
  21  * Boehm-Demers-Weiser conservative garbage collector.
  22  */
  23
  24 #include <stdint.h>
  25 #include <urcu/compiler.h>
  26 #include <urcu/system.h>
  27
  28 #ifdef __cplusplus
  29 extern "C" {
  30 #endif
  31
  32 #ifndef uatomic_set
  33 #define uatomic_set(addr, v)    ((void) CMM_STORE_SHARED(*(addr), (v)))
  34 #endif
  35
  36 #ifndef uatomic_read
  37 #define uatomic_read(addr)      CMM_LOAD_SHARED(*(addr))
  38 #endif
  39
  40 #if !defined __OPTIMIZE__  || defined UATOMIC_NO_LINK_ERROR
  41 static inline __attribute__((always_inline))
  42 void _uatomic_link_error(void)
  43 {
  44 #ifdef ILLEGAL_INSTR
  45         /*
  46          * generate an illegal instruction. Cannot catch this with
  47          * linker tricks when optimizations are disabled.
  48          */
  49         __asm__ __volatile__(ILLEGAL_INSTR);
  50 #else
  51         __builtin_trap();
  52 #endif
  53 }
  54
  55 #else /* #if !defined __OPTIMIZE__  || defined UATOMIC_NO_LINK_ERROR */
  56 extern void _uatomic_link_error(void);
  57 #endif /* #else #if !defined __OPTIMIZE__  || defined UATOMIC_NO_LINK_ERROR */
  58
  59 /* cmpxchg */
  60
  61 #ifndef uatomic_cmpxchg
  62 static inline __attribute__((always_inline))
  63 unsigned long _uatomic_cmpxchg(void *addr, unsigned long old,
  64                               unsigned long _new, int len)
  65 {
  66         switch (len) {
  67 #ifdef UATOMIC_HAS_ATOMIC_BYTE
  68         case 1:
  69                 return __sync_val_compare_and_swap_1((uint8_t *) addr, old,
  70                                 _new);
  71 #endif
  72 #ifdef UATOMIC_HAS_ATOMIC_SHORT
  73         case 2:
  74                 return __sync_val_compare_and_swap_2((uint16_t *) addr, old,
  75                                 _new);
  76 #endif
  77         case 4:
  78                 return __sync_val_compare_and_swap_4((uint32_t *) addr, old,
  79                                 _new);
  80 #if (CAA_BITS_PER_LONG == 64)
  81         case 8:
  82                 return __sync_val_compare_and_swap_8((uint64_t *) addr, old,
  83                                 _new);
  84 #endif
  85         }
  86         _uatomic_link_error();
  87         return 0;
  88 }
  89
  90
  91 #define uatomic_cmpxchg(addr, old, _new)                                      \
  92         ((__typeof__(*(addr))) _uatomic_cmpxchg((addr),                       \
  93                                                 caa_cast_long_keep_sign(old), \
  94                                                 caa_cast_long_keep_sign(_new),\
  95                                                 sizeof(*(addr))))
  96
  97
  98 /* uatomic_and */
  99
 100 #ifndef uatomic_and
 101 static inline __attribute__((always_inline))
 102 void _uatomic_and(void *addr, unsigned long val,
 103                   int len)
 104 {
 105         switch (len) {
 106 #ifdef UATOMIC_HAS_ATOMIC_BYTE
 107         case 1:
 108                 __sync_and_and_fetch_1((uint8_t *) addr, val);
 109                 return;
 110 #endif
 111 #ifdef UATOMIC_HAS_ATOMIC_SHORT
 112         case 2:
 113                 __sync_and_and_fetch_2((uint16_t *) addr, val);
 114                 return;
 115 #endif
 116         case 4:
 117                 __sync_and_and_fetch_4((uint32_t *) addr, val);
 118                 return;
 119 #if (CAA_BITS_PER_LONG == 64)
 120         case 8:
 121                 __sync_and_and_fetch_8((uint64_t *) addr, val);
 122                 return;
 123 #endif
 124         }
 125         _uatomic_link_error();
 126 }
 127
 128 #define uatomic_and(addr, v)                    \
 129         (_uatomic_and((addr),                   \
 130                 caa_cast_long_keep_sign(v),     \
 131                 sizeof(*(addr))))
 132 #define cmm_smp_mb__before_uatomic_and()        cmm_barrier()
 133 #define cmm_smp_mb__after_uatomic_and()         cmm_barrier()
 134
 135 #endif
 136
 137 /* uatomic_or */
 138
 139 #ifndef uatomic_or
 140 static inline __attribute__((always_inline))
 141 void _uatomic_or(void *addr, unsigned long val,
 142                  int len)
 143 {
 144         switch (len) {
 145 #ifdef UATOMIC_HAS_ATOMIC_BYTE
 146         case 1:
 147                 __sync_or_and_fetch_1((uint8_t *) addr, val);
 148                 return;
 149 #endif
 150 #ifdef UATOMIC_HAS_ATOMIC_SHORT
 151         case 2:
 152                 __sync_or_and_fetch_2((uint16_t *) addr, val);
 153                 return;
 154 #endif
 155         case 4:
 156                 __sync_or_and_fetch_4((uint32_t *) addr, val);
 157                 return;
 158 #if (CAA_BITS_PER_LONG == 64)
 159         case 8:
 160                 __sync_or_and_fetch_8((uint64_t *) addr, val);
 161                 return;
 162 #endif
 163         }
 164         _uatomic_link_error();
 165         return;
 166 }
 167
 168 #define uatomic_or(addr, v)                     \
 169         (_uatomic_or((addr),                    \
 170                 caa_cast_long_keep_sign(v),     \
 171                 sizeof(*(addr))))
 172 #define cmm_smp_mb__before_uatomic_or()         cmm_barrier()
 173 #define cmm_smp_mb__after_uatomic_or()          cmm_barrier()
 174
 175 #endif
 176
 177
 178 /* uatomic_add_return */
 179
 180 #ifndef uatomic_add_return
 181 static inline __attribute__((always_inline))
 182 unsigned long _uatomic_add_return(void *addr, unsigned long val,
 183                                  int len)
 184 {
 185         switch (len) {
 186 #ifdef UATOMIC_HAS_ATOMIC_BYTE
 187         case 1:
 188                 return __sync_add_and_fetch_1((uint8_t *) addr, val);
 189 #endif
 190 #ifdef UATOMIC_HAS_ATOMIC_SHORT
 191         case 2:
 192                 return __sync_add_and_fetch_2((uint16_t *) addr, val);
 193 #endif
 194         case 4:
 195                 return __sync_add_and_fetch_4((uint32_t *) addr, val);
 196 #if (CAA_BITS_PER_LONG == 64)
 197         case 8:
 198                 return __sync_add_and_fetch_8((uint64_t *) addr, val);
 199 #endif
 200         }
 201         _uatomic_link_error();
 202         return 0;
 203 }
 204
 205
 206 #define uatomic_add_return(addr, v)                                         \
 207         ((__typeof__(*(addr))) _uatomic_add_return((addr),                  \
 208                                                 caa_cast_long_keep_sign(v), \
 209                                                 sizeof(*(addr))))
 210 #endif /* #ifndef uatomic_add_return */
 211
 212 #ifndef uatomic_xchg
 213 /* xchg */
 214
 215 static inline __attribute__((always_inline))
 216 unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
 217 {
 218         switch (len) {
 219 #ifdef UATOMIC_HAS_ATOMIC_BYTE
 220         case 1:
 221         {
 222                 uint8_t old;
 223
 224                 do {
 225                         old = uatomic_read((uint8_t *) addr);
 226                 } while (!__sync_bool_compare_and_swap_1((uint8_t *) addr,
 227                                 old, val));
 228
 229                 return old;
 230         }
 231 #endif
 232 #ifdef UATOMIC_HAS_ATOMIC_SHORT
 233         case 2:
 234         {
 235                 uint16_t old;
 236
 237                 do {
 238                         old = uatomic_read((uint16_t *) addr);
 239                 } while (!__sync_bool_compare_and_swap_2((uint16_t *) addr,
 240                                 old, val));
 241
 242                 return old;
 243         }
 244 #endif
 245         case 4:
 246         {
 247                 uint32_t old;
 248
 249                 do {
 250                         old = uatomic_read((uint32_t *) addr);
 251                 } while (!__sync_bool_compare_and_swap_4((uint32_t *) addr,
 252                                 old, val));
 253
 254                 return old;
 255         }
 256 #if (CAA_BITS_PER_LONG == 64)
 257         case 8:
 258         {
 259                 uint64_t old;
 260
 261                 do {
 262                         old = uatomic_read((uint64_t *) addr);
 263                 } while (!__sync_bool_compare_and_swap_8((uint64_t *) addr,
 264                                 old, val));
 265
 266                 return old;
 267         }
 268 #endif
 269         }
 270         _uatomic_link_error();
 271         return 0;
 272 }
 273
 274 #define uatomic_xchg(addr, v)                                               \
 275         ((__typeof__(*(addr))) _uatomic_exchange((addr),                    \
 276                                                 caa_cast_long_keep_sign(v), \
 277                                                 sizeof(*(addr))))
 278 #endif /* #ifndef uatomic_xchg */
 279
 280 #else /* #ifndef uatomic_cmpxchg */
 281
 282 #ifndef uatomic_and
 283 /* uatomic_and */
 284
 285 static inline __attribute__((always_inline))
 286 void _uatomic_and(void *addr, unsigned long val, int len)
 287 {
 288         switch (len) {
 289 #ifdef UATOMIC_HAS_ATOMIC_BYTE
 290         case 1:
 291         {
 292                 uint8_t old, oldt;
 293
 294                 oldt = uatomic_read((uint8_t *) addr);
 295                 do {
 296                         old = oldt;
 297                         oldt = _uatomic_cmpxchg(addr, old, old & val, 1);
 298                 } while (oldt != old);
 299
 300                 return;
 301         }
 302 #endif
 303 #ifdef UATOMIC_HAS_ATOMIC_SHORT
 304         case 2:
 305         {
 306                 uint16_t old, oldt;
 307
 308                 oldt = uatomic_read((uint16_t *) addr);
 309                 do {
 310                         old = oldt;
 311                         oldt = _uatomic_cmpxchg(addr, old, old & val, 2);
 312                 } while (oldt != old);
 313         }
 314 #endif
 315         case 4:
 316         {
 317                 uint32_t old, oldt;
 318
 319                 oldt = uatomic_read((uint32_t *) addr);
 320                 do {
 321                         old = oldt;
 322                         oldt = _uatomic_cmpxchg(addr, old, old & val, 4);
 323                 } while (oldt != old);
 324
 325                 return;
 326         }
 327 #if (CAA_BITS_PER_LONG == 64)
 328         case 8:
 329         {
 330                 uint64_t old, oldt;
 331
 332                 oldt = uatomic_read((uint64_t *) addr);
 333                 do {
 334                         old = oldt;
 335                         oldt = _uatomic_cmpxchg(addr, old, old & val, 8);
 336                 } while (oldt != old);
 337
 338                 return;
 339         }
 340 #endif
 341         }
 342         _uatomic_link_error();
 343 }
 344
 345 #define uatomic_and(addr, v)                    \
 346         (_uatomic_and((addr),                   \
 347                 caa_cast_long_keep_sign(v),     \
 348                 sizeof(*(addr))))
 349 #define cmm_smp_mb__before_uatomic_and()        cmm_barrier()
 350 #define cmm_smp_mb__after_uatomic_and()         cmm_barrier()
 351
 352 #endif /* #ifndef uatomic_and */
 353
 354 #ifndef uatomic_or
 355 /* uatomic_or */
 356
 357 static inline __attribute__((always_inline))
 358 void _uatomic_or(void *addr, unsigned long val, int len)
 359 {
 360         switch (len) {
 361 #ifdef UATOMIC_HAS_ATOMIC_BYTE
 362         case 1:
 363         {
 364                 uint8_t old, oldt;
 365
 366                 oldt = uatomic_read((uint8_t *) addr);
 367                 do {
 368                         old = oldt;
 369                         oldt = _uatomic_cmpxchg(addr, old, old | val, 1);
 370                 } while (oldt != old);
 371
 372                 return;
 373         }
 374 #endif
 375 #ifdef UATOMIC_HAS_ATOMIC_SHORT
 376         case 2:
 377         {
 378                 uint16_t old, oldt;
 379
 380                 oldt = uatomic_read((uint16_t *) addr);
 381                 do {
 382                         old = oldt;
 383                         oldt = _uatomic_cmpxchg(addr, old, old | val, 2);
 384                 } while (oldt != old);
 385
 386                 return;
 387         }
 388 #endif
 389         case 4:
 390         {
 391                 uint32_t old, oldt;
 392
 393                 oldt = uatomic_read((uint32_t *) addr);
 394                 do {
 395                         old = oldt;
 396                         oldt = _uatomic_cmpxchg(addr, old, old | val, 4);
 397                 } while (oldt != old);
 398
 399                 return;
 400         }
 401 #if (CAA_BITS_PER_LONG == 64)
 402         case 8:
 403         {
 404                 uint64_t old, oldt;
 405
 406                 oldt = uatomic_read((uint64_t *) addr);
 407                 do {
 408                         old = oldt;
 409                         oldt = _uatomic_cmpxchg(addr, old, old | val, 8);
 410                 } while (oldt != old);
 411
 412                 return;
 413         }
 414 #endif
 415         }
 416         _uatomic_link_error();
 417 }
 418
 419 #define uatomic_or(addr, v)                     \
 420         (_uatomic_or((addr),                    \
 421                 caa_cast_long_keep_sign(v),     \
 422                 sizeof(*(addr))))
 423 #define cmm_smp_mb__before_uatomic_or()         cmm_barrier()
 424 #define cmm_smp_mb__after_uatomic_or()          cmm_barrier()
 425
 426 #endif /* #ifndef uatomic_or */
 427
 428 #ifndef uatomic_add_return
 429 /* uatomic_add_return */
 430
 431 static inline __attribute__((always_inline))
 432 unsigned long _uatomic_add_return(void *addr, unsigned long val, int len)
 433 {
 434         switch (len) {
 435 #ifdef UATOMIC_HAS_ATOMIC_BYTE
 436         case 1:
 437         {
 438                 uint8_t old, oldt;
 439
 440                 oldt = uatomic_read((uint8_t *) addr);
 441                 do {
 442                         old = oldt;
 443                         oldt = uatomic_cmpxchg((uint8_t *) addr,
 444                                                old, old + val);
 445                 } while (oldt != old);
 446
 447                 return old + val;
 448         }
 449 #endif
 450 #ifdef UATOMIC_HAS_ATOMIC_SHORT
 451         case 2:
 452         {
 453                 uint16_t old, oldt;
 454
 455                 oldt = uatomic_read((uint16_t *) addr);
 456                 do {
 457                         old = oldt;
 458                         oldt = uatomic_cmpxchg((uint16_t *) addr,
 459                                                old, old + val);
 460                 } while (oldt != old);
 461
 462                 return old + val;
 463         }
 464 #endif
 465         case 4:
 466         {
 467                 uint32_t old, oldt;
 468
 469                 oldt = uatomic_read((uint32_t *) addr);
 470                 do {
 471                         old = oldt;
 472                         oldt = uatomic_cmpxchg((uint32_t *) addr,
 473                                                old, old + val);
 474                 } while (oldt != old);
 475
 476                 return old + val;
 477         }
 478 #if (CAA_BITS_PER_LONG == 64)
 479         case 8:
 480         {
 481                 uint64_t old, oldt;
 482
 483                 oldt = uatomic_read((uint64_t *) addr);
 484                 do {
 485                         old = oldt;
 486                         oldt = uatomic_cmpxchg((uint64_t *) addr,
 487                                                old, old + val);
 488                 } while (oldt != old);
 489
 490                 return old + val;
 491         }
 492 #endif
 493         }
 494         _uatomic_link_error();
 495         return 0;
 496 }
 497
 498 #define uatomic_add_return(addr, v)                                         \
 499         ((__typeof__(*(addr))) _uatomic_add_return((addr),                  \
 500                                                 caa_cast_long_keep_sign(v), \
 501                                                 sizeof(*(addr))))
 502 #endif /* #ifndef uatomic_add_return */
 503
 504 #ifndef uatomic_xchg
 505 /* xchg */
 506
 507 static inline __attribute__((always_inline))
 508 unsigned long _uatomic_exchange(void *addr, unsigned long val, int len)
 509 {
 510         switch (len) {
 511 #ifdef UATOMIC_HAS_ATOMIC_BYTE
 512         case 1:
 513         {
 514                 uint8_t old, oldt;
 515
 516                 oldt = uatomic_read((uint8_t *) addr);
 517                 do {
 518                         old = oldt;
 519                         oldt = uatomic_cmpxchg((uint8_t *) addr,
 520                                                old, val);
 521                 } while (oldt != old);
 522
 523                 return old;
 524         }
 525 #endif
 526 #ifdef UATOMIC_HAS_ATOMIC_SHORT
 527         case 2:
 528         {
 529                 uint16_t old, oldt;
 530
 531                 oldt = uatomic_read((uint16_t *) addr);
 532                 do {
 533                         old = oldt;
 534                         oldt = uatomic_cmpxchg((uint16_t *) addr,
 535                                                old, val);
 536                 } while (oldt != old);
 537
 538                 return old;
 539         }
 540 #endif
 541         case 4:
 542         {
 543                 uint32_t old, oldt;
 544
 545                 oldt = uatomic_read((uint32_t *) addr);
 546                 do {
 547                         old = oldt;
 548                         oldt = uatomic_cmpxchg((uint32_t *) addr,
 549                                                old, val);
 550                 } while (oldt != old);
 551
 552                 return old;
 553         }
 554 #if (CAA_BITS_PER_LONG == 64)
 555         case 8:
 556         {
 557                 uint64_t old, oldt;
 558
 559                 oldt = uatomic_read((uint64_t *) addr);
 560                 do {
 561                         old = oldt;
 562                         oldt = uatomic_cmpxchg((uint64_t *) addr,
 563                                                old, val);
 564                 } while (oldt != old);
 565
 566                 return old;
 567         }
 568 #endif
 569         }
 570         _uatomic_link_error();
 571         return 0;
 572 }
 573
 574 #define uatomic_xchg(addr, v)                                               \
 575         ((__typeof__(*(addr))) _uatomic_exchange((addr),                    \
 576                                                 caa_cast_long_keep_sign(v), \
 577                                                 sizeof(*(addr))))
 578 #endif /* #ifndef uatomic_xchg */
 579
 580 #endif /* #else #ifndef uatomic_cmpxchg */
 581
 582 /* uatomic_sub_return, uatomic_add, uatomic_sub, uatomic_inc, uatomic_dec */
 583
 584 #ifndef uatomic_add
 585 #define uatomic_add(addr, v)            (void)uatomic_add_return((addr), (v))
 586 #define cmm_smp_mb__before_uatomic_add()        cmm_barrier()
 587 #define cmm_smp_mb__after_uatomic_add()         cmm_barrier()
 588 #endif
 589
 590 #define uatomic_sub_return(addr, v)     \
 591         uatomic_add_return((addr), -(caa_cast_long_keep_sign(v)))
 592 #define uatomic_sub(addr, v)            \
 593         uatomic_add((addr), -(caa_cast_long_keep_sign(v)))
 594 #define cmm_smp_mb__before_uatomic_sub()        cmm_smp_mb__before_uatomic_add()
 595 #define cmm_smp_mb__after_uatomic_sub()         cmm_smp_mb__after_uatomic_add()
 596
 597 #ifndef uatomic_inc
 598 #define uatomic_inc(addr)               uatomic_add((addr), 1)
 599 #define cmm_smp_mb__before_uatomic_inc()        cmm_smp_mb__before_uatomic_add()
 600 #define cmm_smp_mb__after_uatomic_inc()         cmm_smp_mb__after_uatomic_add()
 601 #endif
 602
 603 #ifndef uatomic_dec
 604 #define uatomic_dec(addr)               uatomic_add((addr), -1)
 605 #define cmm_smp_mb__before_uatomic_dec()        cmm_smp_mb__before_uatomic_add()
 606 #define cmm_smp_mb__after_uatomic_dec()         cmm_smp_mb__after_uatomic_add()
 607 #endif
 608
 609 #ifdef __cplusplus
 610 }
 611 #endif
 612
 613 #endif /* _URCU_UATOMIC_GENERIC_H */